In [1]:
import pandas as pd
In [2]:
import numpy as np
In [3]:
import matplotlib.pyplot as plt
In [4]:
%matplotlib inline
In [5]:
from mpl_toolkits import mplot3d

Batch ,Stochastic and mini Gradient Descent

In [6]:
feature=pd.read_csv('training_feature_matrix.csv')

feature.mean()
Out[6]:
f_1      -1.420408
f_2    1326.666667
dtype: float64
In [7]:
feature.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 245 entries, 0 to 244
Data columns (total 2 columns):
f_1    245 non-null float64
f_2    245 non-null float64
dtypes: float64(2)
memory usage: 3.9 KB
In [8]:
output=pd.read_csv('training_output.csv')

output.mean()
Out[8]:
output    16.960068
dtype: float64
In [9]:
feature=pd.read_csv('training_feature_matrix.csv')
In [10]:
feature['constant']=1
features=['f_1','f_2']
features=['constant']+features
feature=feature[features]
feature
Out[10]:
constant f_1 f_2
0 1 2.6 1360.00
1 1 2.0 1292.25
2 1 1.2 1185.00
3 1 2.0 1333.25
4 1 4.8 1580.75
5 1 3.1 1350.25
6 1 2.7 1262.75
7 1 2.1 1206.25
8 1 2.5 1251.50
9 1 2.7 1287.00
10 1 2.9 1352.75
11 1 2.8 1309.00
12 1 2.4 1274.00
13 1 3.9 1509.50
14 1 3.7 1525.25
15 1 6.6 1843.00
16 1 3.5 1483.50
17 1 4.1 1541.50
18 1 3.6 1451.25
19 1 2.8 1328.25
20 1 2.0 1206.50
21 1 2.0 1239.50
22 1 2.5 1306.00
23 1 2.3 1326.00
24 1 3.2 1472.75
25 1 4.2 1609.00
26 1 4.2 1610.75
27 1 4.2 1620.50
28 1 3.1 1444.25
29 1 2.6 1418.00
... ... ... ...
215 1 2.2 1099.50
216 1 2.1 1093.75
217 1 2.3 1075.50
218 1 2.6 1151.75
219 1 3.8 1406.75
220 1 4.4 1437.50
221 1 4.5 1424.50
222 1 3.3 1278.50
223 1 2.0 1131.75
224 1 1.1 1006.25
225 1 0.9 982.25
226 1 0.8 968.50
227 1 0.7 940.50
228 1 3.8 1498.25
229 1 3.0 1308.00
230 1 2.7 1209.00
231 1 3.5 1310.25
232 1 2.8 1170.75
233 1 1.9 1112.00
234 1 3.0 1225.00
235 1 1.9 1064.50
236 1 2.7 1236.25
237 1 1.7 1089.00
238 1 1.7 1137.75
239 1 1.5 1097.50
240 1 1.7 1117.50
241 1 2.2 1170.25
242 1 4.5 1501.25
243 1 3.5 1272.50
244 1 1.0 969.50

245 rows × 3 columns

In [11]:
def feature_normalize(data):
    data=(data-data.mean())/data.std()
    return data
In [12]:
def cost_function(feature,weights,output):
    cost=0
    cost_half=0
    
    m=feature.shape[0]
    n=feature.shape[1]
    y_hat=np.zeros(m)
    for i in range(m):
        y_hat=0
        for j in range(n):
            
            y_hat+=weights[j]*feature[i][j]
        error=output[i]-y_hat
        cost_half+=error*error
    cost=(0.5*cost_half)/m
    
    return cost
In [ ]:
 
In [13]:
def bgd(feature,output,alpha,iteration,weights):
    m=feature.shape[0]
    n=feature.shape[1]
    feature=feature.to_numpy()
    output=output.to_numpy()
    feature=feature_normalize(feature)
    cost=np.zeros(iteration)
    weights=np.ones(n)
    v_1= np.zeros(iteration)
    v_2= np.zeros(iteration)
    v_3= np.zeros(iteration)
    y=np.zeros(m)
    for it in range(iteration):
        temp=weights
        
        gradient=np.zeros(n)
        for i in range(m):
            y_hat=0
            for j in range(n):
                y_hat+=temp[j]*feature[i][j]
            error=output[i]-y_hat
            for j in range(n):
                gradient[j]+=error*feature[i][j]
            
        for j in range(n):
            temp[j]=temp[j]+(alpha*gradient[j])/m
        weights=temp
        
        v_1[it]=weights[0]
        v_2[it]=weights[1]
        v_3[it]=weights[2]
        cost[it]=cost_function(feature,weights,output)
    fig = plt.figure()
    ax = plt.axes(projection='3d')
    ax.plot3D(v_1,v_2,v_3,'gray')
    ax.set_title('3D line plot')
    plt.show()
    
    plt.plot(cost)

    return (weights,cost)

               
In [14]:
def sgd(feature,output,alpha,iteration,weights):
    m=feature.shape[0]
    n=feature.shape[1]
    feature=feature.to_numpy()
    output=output.to_numpy()
    feature=feature_normalize(feature)
    cost=np.zeros(iteration)
    weights=np.ones(n)
    v_1= np.zeros(iteration)
    v_2= np.zeros(iteration)
    v_3= np.zeros(iteration)
    y=np.zeros(m)
    for it in range(iteration):
        temp=weights
        
        gradient=np.zeros(n)
        for i in range(m):
            y_hat=0
            temp=weights
            for j in range(n):
                y_hat+=temp[j]*feature[i][j]
            error=output[i]-y_hat
            for j in range(n):
                gradient[j]=error*feature[i][j]
            
            for j in range(n):
                temp[j]=temp[j]+(alpha*gradient[j])
            weights=temp
        
            v_1[i]=weights[0]
            v_2[i]=weights[1]
            v_3[i]=weights[2]
        cost[it]=cost_function(feature,weights,output)
    fig = plt.figure()
    ax = plt.axes(projection='3d')
    ax.plot3D(v_1,v_2,v_3,'gray')
    ax.set_title('3D line plot')
    plt.show()
    
    plt.plot(cost)

    return (weights,cost)

               
In [15]:
def mbgd(feature,output,alpha,iteration,weights,x):
    m=feature.shape[0]
    n=feature.shape[1]
    feature=feature.to_numpy()
    output=output.to_numpy()
    feature=feature_normalize(feature)
    cost=np.zeros(iteration)
    weights=np.ones(n)
    v_1= np.zeros(iteration)
    v_2= np.zeros(iteration)
    v_3= np.zeros(iteration)
    y=np.zeros(m)
    for it in range(iteration):
        temp=weights
        
        gradient=np.zeros(n)
        for i in range((m-x)/10):
            temp=weights
            gradient=np.zeros(n)
            i=i*x
            for z in range(i,i+x):
                y_hat=0
                for j in range(n):
                    
                    y_hat+=temp[j]*feature[z][j]
                error=output[z]-y_hat
                for j in range(n):
                    gradient[j]+=error*feature[z][j]

            for j in range(n):
                temp[j]=temp[j]+(alpha*gradient[j])/x
            weights=temp
        
            v_1[i]=weights[0]
            v_2[i]=weights[1]
            v_3[i]=weights[2]
        cost[it]=cost_function(feature,weights,output)
    fig = plt.figure()
    ax = plt.axes(projection='3d')
    ax.plot3D(v_1,v_2,v_3,'gray')
    ax.set_title('3D line plot')
    plt.show()
    
    plt.plot(cost)

    return (weights,cost)

               
In [16]:
output=pd.read_csv('training_output.csv')
In [17]:
weights=np.zeros(3)
(weights,cost)=mbgd(feature,output,0.0001,1000,weights,5)
print weights
[-3.25619185 -3.28134978  8.45062871]
In [18]:
weights=np.zeros(3)
(weights,cost)=sgd(feature,output,0.001,1000,weights)
print weights
[-11.14600013  -6.81512766   3.29885439]
In [19]:
weights=np.zeros(3)
(weights,cost)=bgd(feature,output,0.001,1000,weights)
print weights
[-2.94154358 -2.96836707  8.41814008]
In [ ]:
 
In [20]:
def predict(feature,weights,output):
    m=feature.shape[0]
    n=feature.shape[1]
    error_square=0
    for i in range(m):
        y_hat=0
        for j in range(n):
            y_hat+=weights[j]*feature[i][j]
        
        error_square+=(y_hat-output[i])**2
    error=error_square/m
    return error
In [21]:
test_feature=pd.read_csv('test_feature_matrix.csv')
test_output=pd.read_csv('test_output.csv')
test_feature['constant']=1
features=['f_1','f_2']
features=['constant']+features
test_feature=test_feature[features]
test_feature
feature=test_feature.to_numpy()
output=test_output.to_numpy()
feature=feature_normalize(feature)
In [22]:
print(predict(feature,weights,output))
[42.48592152]

Batch ,Stochastic and mini Gradient Descent using ridge regression :>

In [ ]:
 
In [23]:
def cost_function(feature,weights,output,l_2):
    cost=0
    cost_half=0
    
    m=feature.shape[0]
    n=feature.shape[1]
    y_hat=np.zeros(m)
    for i in range(m):
        y_hat=0
        x=0
        for j in range(n):
            
            y_hat+=weights[j]*feature[i][j]
            x+=l_2*(weights[j]**2)
        error=output[i]-(y_hat)
        cost_half+=error*error+x
    cost=(0.5*cost_half)/m
    
    return cost
In [24]:
def bgd(feature,output,alpha,iteration,weights,reg_cons):
    m=feature.shape[0]
    n=feature.shape[1]
    
    feature=feature_normalize(feature)
    cost=np.zeros(iteration)
    weights=np.ones(n)
    v_1= np.zeros(iteration)
    v_2= np.zeros(iteration)
    v_3= np.zeros(iteration)
    y=np.zeros(m)
    for it in range(iteration):
        temp=weights
        
        gradient=np.zeros(n)
        for i in range(m):
            y_hat=0
            x=0
            for j in range(n):
                y_hat+=temp[j]*feature[i][j]
            error=output[i]-y_hat
            for j in range(n):
                x+=reg_cons*temp[j]
                gradient[j]+=error*feature[i][j]+x
            
        for j in range(n):
            temp[j]=temp[j]+(alpha*gradient[j])/m
        weights=temp
        
        v_1[it]=weights[0]
        v_2[it]=weights[1]
        v_3[it]=weights[2]
        cost[it]=cost_function(feature,weights,output,reg_cons)
    
    plt.plot(cost)
    
    

    return (weights,cost)

               
In [25]:
weights=np.zeros(3)
(weights,cost)=bgd(feature,output,0.00001,500,weights,0.001)
print weights
[0.96660572 0.96468153 1.0683475 ]
In [26]:
def sgd(feature,output,alpha,iteration,weights,reg_cons):
    m=feature.shape[0]
    n=feature.shape[1]
    
    feature=feature_normalize(feature)
    cost=np.zeros(iteration)
    weights=np.ones(n)
    v_1= np.zeros(iteration)
    v_2= np.zeros(iteration)
    v_3= np.zeros(iteration)
    y=np.zeros(m)
    for it in range(iteration):
        temp=weights
        
        gradient=np.zeros(n)
        for i in range(m):
            y_hat=0
            x=0
            temp=weights
            for j in range(n):
                y_hat+=temp[j]*feature[i][j]
            error=output[i]-y_hat
            for j in range(n):
                x+=reg_cons*temp[j]
                gradient[j]=error*feature[i][j]
            
            for j in range(n):
                temp[j]=temp[j]+(alpha*gradient[j])
            weights=temp
        
            v_1[i]=weights[0]
            v_2[i]=weights[1]
            v_3[i]=weights[2]
        cost[it]=cost_function(feature,weights,output,reg_cons)
    fig = plt.figure()
    ax = plt.axes(projection='3d')
    ax.plot3D(v_1,v_2,v_3,'gray')
    ax.set_title('3D line plot')
    plt.show()
    
    plt.plot(cost)

    return (weights,cost)

               
In [27]:
weights=np.zeros(3)
(weights,cost)=sgd(feature,output,0.0001,1000,weights,0.1)
print weights
[-3.25039382 -3.61068024  3.73714297]
In [28]:
def mbgd(feature,output,alpha,iteration,weights,x,reg_cons):
    m=feature.shape[0]
    n=feature.shape[1]
    
    feature=feature_normalize(feature)
    cost=np.zeros(iteration)
    weights=np.ones(n)
    v_1= np.zeros(iteration)
    v_2= np.zeros(iteration)
    v_3= np.zeros(iteration)
    y=np.zeros(m)
    for it in range(iteration):
        temp=weights
        
        gradient=np.zeros(n)
        for i in range((m-x)/10):
            temp=weights
            gradient=np.zeros(n)
            i=i*x
            y=0
            for z in range(i,i+x):
                y_hat=0
                for j in range(n):
                    
                    y_hat+=temp[j]*feature[z][j]
                error=output[z]-y_hat
                for j in range(n):
                    y+=reg_cons*temp[j]
                    gradient[j]+=error*feature[z][j]

            for j in range(n):
                temp[j]=temp[j]+(alpha*gradient[j])/x
            weights=temp
        
            v_1[i]=weights[0]
            v_2[i]=weights[1]
            v_3[i]=weights[2]
        cost[it]=cost_function(feature,weights,output,reg_cons)
    fig = plt.figure()
    ax = plt.axes(projection='3d')
    ax.plot3D(v_1,v_2,v_3,'gray')
    ax.set_title('3D line plot')
    plt.show()
    
    plt.plot(cost)

    return (weights,cost)

               
In [29]:
weights=np.zeros(3)
(weights,cost)=mbgd(feature,output,0.0001,500,weights,5,0.001)
print weights
[-0.47248896 -0.53106889  4.12545311]
In [30]:
def predict(feature,weights,output):
    m=feature.shape[0]
    n=feature.shape[1]
    error_square=0
    for i in range(m):
        y_hat=0
        for j in range(n):
            y_hat+=weights[j]*feature[i][j]
        
        error_square+=(y_hat-output[i])**2
    error=error_square/m
    return error
In [31]:
test_feature=pd.read_csv('test_feature_matrix.csv')
test_output=pd.read_csv('test_output.csv')
test_feature['constant']=1
features=['f_1','f_2']
features=['constant']+features
test_feature=test_feature[features]
test_feature
feature=test_feature.to_numpy()
output=test_output.to_numpy()
feature=feature_normalize(feature)
In [32]:
print(predict(feature,weights,output))
[18.14545105]

Derivation of least angle Regression

IMG_20201010_001149.jpg

IMG_20201010_001208.jpg

IMG_20201010_001225.jpg

Least angle regression

In [33]:
def lar_cost_function(feature,weights,output,l_1):
    cost=0
    cost_half=0
    
    m=feature.shape[0]
    n=feature.shape[1]
    y_hat=np.zeros(m)
    
    for i in range(m):
        y_hat=0
        x=0
        for j in range(n):
            
            y_hat+=weights[j]*feature[i][j]
            x+=l_1*weights[j]
        error=output[i]-y_hat
        cost_half+=error*error
    cost=((0.5*cost_half)/m)+x/2
    
    return cost
In [34]:
def lar_bgd(feature,output,alpha,iteration,weights,l_1):
    m=feature.shape[0]
    n=feature.shape[1]
    
    feature=feature_normalize(feature)
    cost=np.zeros(iteration)
    weights=np.ones(n)
    v_1= np.zeros(iteration)
    v_2= np.zeros(iteration)
    v_3= np.zeros(iteration)
    y=np.zeros(m)
    x=0
    for it in range(iteration):
        temp=weights
        
        gradient=np.zeros(n)
        for i in range(m):
            y_hat=0
            
            for j in range(n):
                y_hat+=temp[j]*feature[i][j]
            error=output[i]-y_hat
            for j in range(n):
                gradient[j]+=error*feature[i][j]
                x+=l_1*np.sign(weights[j])
            
        for j in range(n):
            temp[j]=temp[j]+(alpha*(gradient[j]+x))/m
        weights=temp
        
        v_1[it]=weights[0]
        v_2[it]=weights[1]
        v_3[it]=weights[2]
        cost[it]=lar_cost_function(feature,weights,output,l_1)
    fig = plt.figure()
    ax = plt.axes(projection='3d')
    ax.plot3D(v_1,v_2,v_3,'gray')
    ax.set_title('3D line plot')
    plt.show()
    
    plt.plot(cost)

    return (weights,cost)

               
In [ ]:
 
In [35]:
weights=np.zeros(3)
(weights,cost)=lar_bgd(feature,output,0.0001,1000,weights,0.001)
print weights
[0.56369731 0.52928599 2.33934225]
In [36]:
def lar_sgd(feature,output,alpha,iteration,weights,l_1):
    m=feature.shape[0]
    n=feature.shape[1]
    
    feature=feature_normalize(feature)
    cost=np.zeros(iteration)
    weights=np.ones(n)
    v_1= np.zeros(iteration)
    v_2= np.zeros(iteration)
    v_3= np.zeros(iteration)
    y=np.zeros(m)
    x=0
    for it in range(iteration):
        temp=weights
        
        gradient=np.zeros(n)
        for i in range(m):
            y_hat=0
            temp=weights
            x=0
            for j in range(n):
                y_hat+=temp[j]*feature[i][j]
            error=output[i]-y_hat
            for j in range(n):
                gradient[j]=error*feature[i][j]
                x+=l_1*np.sign(weights[j])
            for j in range(n):
                temp[j]=temp[j]+(alpha*(gradient[j]+x))
            weights=temp
        
            v_1[i]=weights[0]
            v_2[i]=weights[1]
            v_3[i]=weights[2]
        cost[it]=lar_cost_function(feature,weights,output,l_1)
    fig = plt.figure()
    ax = plt.axes(projection='3d')
    ax.plot3D(v_1,v_2,v_3,'gray')
    ax.set_title('3D line plot')
    plt.show()
    
    plt.plot(cost)

    return (weights,cost)

               
In [37]:
weights=np.zeros(3)
(weights,cost)=lar_sgd(feature,output,0.0001,1000,weights,0.1)
print weights
[-4.03409769 -4.30942375  3.03465104]
In [38]:
def lar_mbgd(feature,output,alpha,iteration,weights,x,l_1):
    m=feature.shape[0]
    n=feature.shape[1]
    
    feature=feature_normalize(feature)
    cost=np.zeros(iteration)
    weights=np.ones(n)
    v_1= np.zeros(iteration)
    v_2= np.zeros(iteration)
    v_3= np.zeros(iteration)
    y=np.zeros(m)
    for it in range(iteration):
        temp=weights
        
        gradient=np.zeros(n)
        for i in range((m-x)/10):
            temp=weights
            gradient=np.zeros(n)
            i=i*x
            for z in range(i,i+x):
                y_hat=0
                l=0
                for j in range(n):
                    
                    y_hat+=temp[j]*feature[z][j]
                error=output[z]-y_hat
                for j in range(n):
                    gradient[j]+=error*feature[z][j]
                    l+=l_1*np.sign(weights[j])
            for j in range(n):
                temp[j]=temp[j]+(alpha*(gradient[j]+l))/x
            weights=temp
        
            v_1[i]=weights[0]
            v_2[i]=weights[1]
            v_3[i]=weights[2]
        cost[it]=lar_cost_function(feature,weights,output,l_1)
    fig = plt.figure()
    ax = plt.axes(projection='3d')
    ax.plot3D(v_1,v_2,v_3,'gray')
    ax.set_title('3D line plot')
    plt.show()
    
    plt.plot(cost)

    return (weights,cost)

               
In [39]:
weights=np.zeros(3)
(weights,cost)=lar_mbgd(feature,output,0.0001,1000,weights,10,0.01)
print weights
[-1.23428324 -1.39161176  5.17478029]

Vectorized regression

In [40]:
def vect_cost_function(feature,weights,output):
    m=feature.shape[0]
    y_hat=np.dot(np.transpose(weights),np.transpose(feature))
    error=y_hat-output
    cost=(0.5*np.dot(np.transpose(error),error))/m
    return cost,error
In [41]:
def vect_bgd(feature,output,weights):
    m=feature.shape[0]
    
    part_1=np.dot(np.transpose(feature),feature)
    part_2=np.linalg.inv(part_1)
    part_3=np.dot(part_2,np.transpose(feature))
    weights=np.dot(part_3,output)
    cost=vect_cost_function(feature,weights,output)
    
    return weights,cost
In [42]:
weights=np.zeros(3)
(weights,cost)=vect_bgd(feature,output,weights)
print weights
[[-12.33563725]
 [ -1.01064465]
 [  0.80470535]]
In [46]:
def vect_cost_function(feature,weights,output,l):
    m=feature.shape[0]
    y_hat=np.dot(np.transpose(weights),np.transpose(feature))
    error=y_hat-output
    cost=((0.5*np.dot(np.transpose(error),error))/m)+(l*np.dot(np.transpose(weights),weights))/2
    return cost,error
In [47]:
def vect_ridge(feature,output,weights,l):
    m=feature.shape[0]
    
    part_1=np.dot(np.transpose(feature),feature)+l
    part_2=np.linalg.inv(part_1)
    part_3=np.dot(part_2,np.transpose(feature))
    weights=np.dot(part_3,output)
    cost=vect_cost_function(feature,weights,output,l)
    
    return weights,cost
In [48]:
weights=np.zeros(3)
(weights,cost)=vect_ridge(feature,output,weights,0.01)
print weights
[[-12.26025281]
 [ -1.03271882]
 [  0.82877599]]
In [52]:
def vect_cost_function(feature,weights,output,l_1):
    
    m=feature.shape[0]
    y_hat=np.dot(np.transpose(weights),np.transpose(feature))
    
    error=y_hat-output
    cost=((0.5*np.dot(np.transpose(error),error))/m)+(l_1*weights)/2
    return cost,error
In [53]:
def vect_lar(feature,output,weights,l_1):
    m=feature.shape[0]
    
    part_1=np.dot(np.transpose(feature),feature)
    part_2=np.linalg.inv(part_1)
    part_3=np.dot(np.transpose(feature),output)-(l_1*np.sign(weights)*0.5)
    
    weights=np.dot(part_2,part_3)
    print weights
    cost=vect_cost_function(feature,weights,output,l_1)
    plt.plot(cost)
    return weights
In [ ]:
weights=np.ones(3)
(weights,cost)=vect_lar(feature,output,weights,0.1)
print weights
In [ ]:
 

Least angle regression have low rmse as compared to linear regression

Logistic Regression

In [55]:
df=pd.read_csv('data3.csv')
In [56]:
df
Out[56]:
w1 w2 w3 w4 t
0 5.1 3.5 1.4 0.2 1
1 4.9 3.0 1.4 0.2 1
2 4.7 3.2 1.3 0.2 1
3 4.6 3.1 1.5 0.2 1
4 5.0 3.6 1.4 0.2 1
5 5.4 3.9 1.7 0.4 1
6 4.6 3.4 1.4 0.3 1
7 5.0 3.4 1.5 0.2 1
8 4.4 2.9 1.4 0.2 1
9 4.9 3.1 1.5 0.1 1
10 5.4 3.7 1.5 0.2 1
11 4.8 3.4 1.6 0.2 1
12 4.8 3.0 1.4 0.1 1
13 4.3 3.0 1.1 0.1 1
14 5.8 4.0 1.2 0.2 1
15 5.7 4.4 1.5 0.4 1
16 5.4 3.9 1.3 0.4 1
17 5.1 3.5 1.4 0.3 1
18 5.7 3.8 1.7 0.3 1
19 5.1 3.8 1.5 0.3 1
20 5.4 3.4 1.7 0.2 1
21 5.1 3.7 1.5 0.4 1
22 4.6 3.6 1.0 0.2 1
23 5.1 3.3 1.7 0.5 1
24 4.8 3.4 1.9 0.2 1
25 5.0 3.0 1.6 0.2 1
26 5.0 3.4 1.6 0.4 1
27 5.2 3.5 1.5 0.2 1
28 5.2 3.4 1.4 0.2 1
29 4.7 3.2 1.6 0.2 1
... ... ... ... ... ...
70 5.9 3.2 4.8 1.8 2
71 6.1 2.8 4.0 1.3 2
72 6.3 2.5 4.9 1.5 2
73 6.1 2.8 4.7 1.2 2
74 6.4 2.9 4.3 1.3 2
75 6.6 3.0 4.4 1.4 2
76 6.8 2.8 4.8 1.4 2
77 6.7 3.0 5.0 1.7 2
78 6.0 2.9 4.5 1.5 2
79 5.7 2.6 3.5 1.0 2
80 5.5 2.4 3.8 1.1 2
81 5.5 2.4 3.7 1.0 2
82 5.8 2.7 3.9 1.2 2
83 6.0 2.7 5.1 1.6 2
84 5.4 3.0 4.5 1.5 2
85 6.0 3.4 4.5 1.6 2
86 6.7 3.1 4.7 1.5 2
87 6.3 2.3 4.4 1.3 2
88 5.6 3.0 4.1 1.3 2
89 5.5 2.5 4.0 1.3 2
90 5.5 2.6 4.4 1.2 2
91 6.1 3.0 4.6 1.4 2
92 5.8 2.6 4.0 1.2 2
93 5.0 2.3 3.3 1.0 2
94 5.6 2.7 4.2 1.3 2
95 5.7 3.0 4.2 1.2 2
96 5.7 2.9 4.2 1.3 2
97 6.2 2.9 4.3 1.3 2
98 5.1 2.5 3.0 1.1 2
99 5.7 2.8 4.1 1.3 2

100 rows × 5 columns

In [57]:
df = pd.DataFrame(df) 
df.insert(0, "constant", [1]*100, True) 
df
Out[57]:
constant w1 w2 w3 w4 t
0 1 5.1 3.5 1.4 0.2 1
1 1 4.9 3.0 1.4 0.2 1
2 1 4.7 3.2 1.3 0.2 1
3 1 4.6 3.1 1.5 0.2 1
4 1 5.0 3.6 1.4 0.2 1
5 1 5.4 3.9 1.7 0.4 1
6 1 4.6 3.4 1.4 0.3 1
7 1 5.0 3.4 1.5 0.2 1
8 1 4.4 2.9 1.4 0.2 1
9 1 4.9 3.1 1.5 0.1 1
10 1 5.4 3.7 1.5 0.2 1
11 1 4.8 3.4 1.6 0.2 1
12 1 4.8 3.0 1.4 0.1 1
13 1 4.3 3.0 1.1 0.1 1
14 1 5.8 4.0 1.2 0.2 1
15 1 5.7 4.4 1.5 0.4 1
16 1 5.4 3.9 1.3 0.4 1
17 1 5.1 3.5 1.4 0.3 1
18 1 5.7 3.8 1.7 0.3 1
19 1 5.1 3.8 1.5 0.3 1
20 1 5.4 3.4 1.7 0.2 1
21 1 5.1 3.7 1.5 0.4 1
22 1 4.6 3.6 1.0 0.2 1
23 1 5.1 3.3 1.7 0.5 1
24 1 4.8 3.4 1.9 0.2 1
25 1 5.0 3.0 1.6 0.2 1
26 1 5.0 3.4 1.6 0.4 1
27 1 5.2 3.5 1.5 0.2 1
28 1 5.2 3.4 1.4 0.2 1
29 1 4.7 3.2 1.6 0.2 1
... ... ... ... ... ... ...
70 1 5.9 3.2 4.8 1.8 2
71 1 6.1 2.8 4.0 1.3 2
72 1 6.3 2.5 4.9 1.5 2
73 1 6.1 2.8 4.7 1.2 2
74 1 6.4 2.9 4.3 1.3 2
75 1 6.6 3.0 4.4 1.4 2
76 1 6.8 2.8 4.8 1.4 2
77 1 6.7 3.0 5.0 1.7 2
78 1 6.0 2.9 4.5 1.5 2
79 1 5.7 2.6 3.5 1.0 2
80 1 5.5 2.4 3.8 1.1 2
81 1 5.5 2.4 3.7 1.0 2
82 1 5.8 2.7 3.9 1.2 2
83 1 6.0 2.7 5.1 1.6 2
84 1 5.4 3.0 4.5 1.5 2
85 1 6.0 3.4 4.5 1.6 2
86 1 6.7 3.1 4.7 1.5 2
87 1 6.3 2.3 4.4 1.3 2
88 1 5.6 3.0 4.1 1.3 2
89 1 5.5 2.5 4.0 1.3 2
90 1 5.5 2.6 4.4 1.2 2
91 1 6.1 3.0 4.6 1.4 2
92 1 5.8 2.6 4.0 1.2 2
93 1 5.0 2.3 3.3 1.0 2
94 1 5.6 2.7 4.2 1.3 2
95 1 5.7 3.0 4.2 1.2 2
96 1 5.7 2.9 4.2 1.3 2
97 1 6.2 2.9 4.3 1.3 2
98 1 5.1 2.5 3.0 1.1 2
99 1 5.7 2.8 4.1 1.3 2

100 rows × 6 columns

In [58]:
train=df.sample(frac=0.6,random_state=200) #random state is a seed value
test=df.drop(train.index)
train['t'] = (train['t'] < 2)*1
train=train.to_numpy()
train
Out[58]:
array([[1. , 4.9, 3.1, 1.5, 0.1, 1. ],
       [1. , 4.7, 3.2, 1.3, 0.2, 1. ],
       [1. , 6.3, 2.5, 4.9, 1.5, 0. ],
       [1. , 4.9, 3.1, 1.5, 0.1, 1. ],
       [1. , 6.7, 3.1, 4.7, 1.5, 0. ],
       [1. , 5.6, 2.9, 3.6, 1.3, 0. ],
       [1. , 5.5, 2.4, 3.8, 1.1, 0. ],
       [1. , 6.1, 2.9, 4.7, 1.4, 0. ],
       [1. , 4.9, 3.1, 1.5, 0.1, 1. ],
       [1. , 5.1, 3.8, 1.5, 0.3, 1. ],
       [1. , 4.5, 2.3, 1.3, 0.3, 1. ],
       [1. , 5.4, 3.7, 1.5, 0.2, 1. ],
       [1. , 5. , 2.3, 3.3, 1. , 0. ],
       [1. , 5.1, 3.8, 1.6, 0.2, 1. ],
       [1. , 5.3, 3.7, 1.5, 0.2, 1. ],
       [1. , 5.6, 3. , 4.1, 1.3, 0. ],
       [1. , 5. , 2. , 3.5, 1. , 0. ],
       [1. , 5.1, 3.8, 1.9, 0.4, 1. ],
       [1. , 5. , 3.5, 1.6, 0.6, 1. ],
       [1. , 4.3, 3. , 1.1, 0.1, 1. ],
       [1. , 5.1, 3.4, 1.5, 0.2, 1. ],
       [1. , 6. , 3.4, 4.5, 1.6, 0. ],
       [1. , 5.1, 3.5, 1.4, 0.3, 1. ],
       [1. , 4.6, 3.2, 1.4, 0.2, 1. ],
       [1. , 5.2, 3.4, 1.4, 0.2, 1. ],
       [1. , 5.6, 3. , 4.5, 1.5, 0. ],
       [1. , 5.4, 3.9, 1.7, 0.4, 1. ],
       [1. , 6.7, 3. , 5. , 1.7, 0. ],
       [1. , 5.5, 3.5, 1.3, 0.2, 1. ],
       [1. , 4.4, 3. , 1.3, 0.2, 1. ],
       [1. , 6. , 2.9, 4.5, 1.5, 0. ],
       [1. , 6.6, 3. , 4.4, 1.4, 0. ],
       [1. , 6.4, 2.9, 4.3, 1.3, 0. ],
       [1. , 5.6, 2.7, 4.2, 1.3, 0. ],
       [1. , 4.7, 3.2, 1.6, 0.2, 1. ],
       [1. , 5.5, 2.3, 4. , 1.3, 0. ],
       [1. , 4.8, 3.1, 1.6, 0.2, 1. ],
       [1. , 5. , 3.5, 1.3, 0.3, 1. ],
       [1. , 5. , 3. , 1.6, 0.2, 1. ],
       [1. , 5.8, 2.7, 3.9, 1.2, 0. ],
       [1. , 6.7, 3.1, 4.4, 1.4, 0. ],
       [1. , 5.5, 2.4, 3.7, 1. , 0. ],
       [1. , 6.2, 2.9, 4.3, 1.3, 0. ],
       [1. , 5.7, 3.8, 1.7, 0.3, 1. ],
       [1. , 5. , 3.3, 1.4, 0.2, 1. ],
       [1. , 5. , 3.6, 1.4, 0.2, 1. ],
       [1. , 4.8, 3. , 1.4, 0.1, 1. ],
       [1. , 6.1, 2.8, 4. , 1.3, 0. ],
       [1. , 6.1, 3. , 4.6, 1.4, 0. ],
       [1. , 6.3, 2.3, 4.4, 1.3, 0. ],
       [1. , 5.7, 2.8, 4.1, 1.3, 0. ],
       [1. , 6.6, 2.9, 4.6, 1.3, 0. ],
       [1. , 4.4, 2.9, 1.4, 0.2, 1. ],
       [1. , 5.5, 4.2, 1.4, 0.2, 1. ],
       [1. , 5.5, 2.6, 4.4, 1.2, 0. ],
       [1. , 5.6, 2.5, 3.9, 1.1, 0. ],
       [1. , 5.7, 3. , 4.2, 1.2, 0. ],
       [1. , 4.6, 3.1, 1.5, 0.2, 1. ],
       [1. , 5.1, 3.5, 1.4, 0.2, 1. ],
       [1. , 6.9, 3.1, 4.9, 1.5, 0. ]])
In [ ]:
 
In [59]:
def sigmoid(x):
    y=(1/(1+np.exp(-x)))
    return y
x=np.linspace(-10,10,100)
y=sigmoid(x)
plt.plot(x,y)
Out[59]:
[<matplotlib.lines.Line2D at 0x7fe012c74f10>]
In [62]:
fig=plt.figure(figsize=(8, 4))
ax=fig.add_subplot(111)
ax.plot(x, y, linewidth=2, color='#483d8b')
Out[62]:
[<matplotlib.lines.Line2D at 0x7fe01285d6d0>]
In [63]:
fig = plt.figure(figsize=(6, 6))
ax = fig.add_subplot(111)

# Colors for scatter points
colors = ['#483d8b', '#cc8400','#483d8b','#cc8400']

colors_data = [colors[int(i)] for i in y]

# Plot sample data
ax.scatter(train[:,2],train[:,1], color=colors_data, s=100, edgecolor='black', linewidth=2, alpha=0.7)

# Add grid
ax.grid(linestyle='--', linewidth=2, color='gray', alpha=0.2)

# Add axis labels


plt.show()
In [64]:
def feature_normalize(data):
    data=(data-data.mean())/data.std()
    return data
In [65]:
def cost_function(feature,output,weight):
    m=train.shape[0]
    n=(train.shape[1])-1
    cost=np.ones(m)
    cost_func=0
    for i in range(m):
        h=0
        for j in range(n):
            h+=weight[j]*feature[i][j]
        cost[i]=-((output[j])*np.log(sigmoid(h)))-((1-output[i])*np.log(1-sigmoid(h)))
    for k in range(m):
        cost_func+=cost[k]
    net_cost=cost_func/m
    return net_cost
In [66]:
def lgd(alpha,feature,output,weights,iteration):
    
    
    cost=np.ones(iteration)
    m=train.shape[0]
    n=(train.shape[1])-1
    weights=np.ones(n)
    feature=feature_normalize(feature)
    for it in range(iteration):
        temp=weights
        gradient=np.ones(n)
        for i in range(m):
            h=0
            for j in range(n):
                
                h+=temp[j]*feature[i][j]
            error=output[i]-h
            
            for j in range(n):
                gradient[j]+=error*feature[i][j]
                
        for j in range(n):
            temp[j]=temp[j]+(alpha*gradient[j])/m
        weights=temp
        
        cost[it]=cost_function(feature,output,weights)
    plt.plot(cost)
    return weights,cost
In [67]:
w=np.ones(5)
(weight,cost)=lgd(0.18,train[:,0:5],train[:,5],w,300)
print weight
[ 0.97075975  0.64224901  0.56826455 -0.63639474 -0.29859835]
In [68]:
test=test.to_numpy()
In [69]:
def predict(feature,weight):
    
    m=test.shape[0]
    n=(test.shape[1])-1
    predicted=np.zeros(m)
    y=np.zeros(m)
    
    for i in range(m):
        h=0
        for j in range(n):
            h+=weight[j]*feature[i][j]
        predicted[i]=sigmoid(h)
        print predicted[i]
        if(predicted[i]>=0.5):
            y[i]=2
        else:
            y[i]=1
        
    return y,predicted
    
y,predicted=(predict(test[:,0:5], weight))
print y
0.9923994210577672
0.992435325497122
0.9939389485507123
0.9926647624993494
0.9978614125396523
0.9976661785087471
0.9966929416168052
0.9946718730657397
0.9949069740164903
0.994906802135414
0.9925371901737333
0.9911353952171238
0.9931486412371083
0.9949588580345804
0.9950182132483409
0.9965153165905113
0.9943870941927807
0.9912339921347468
0.9916558589723274
0.9796846985623047
0.973088625512153
0.9664698714291089
0.9512358055042701
0.9684275537742012
0.9561824294414545
0.9500167522000698
0.9659100182102187
0.9619572145293099
0.9651860096325158
0.9474102896916826
0.9519515236849398
0.9581232705186002
0.9694232479091754
0.9729616121798069
0.9330927679717954
0.9443820180499675
0.952121854071142
0.9633638199665561
0.9615249502331116
0.968606229362932
[2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2.
 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2.]
In [70]:
m=40
true_positive=0
true_negative=0
false_positive=0
false_negative=0
for i in range(m):
    if(test[i,5]==y[i] and int(y[i])==2):
        true_negative+=1
    if(test[i,5]==y[i] and int(y[i])==1):
        true_positive+=1
    if(test[i,5]!=y[i] and int(y[i])==2):
        false_negative+=1
    if(test[i,5]!=y[i] and int(y[i])==1):
        false_positive+=1
sensitivity=(float(true_positive))/(true_positive+false_negative)
specificity=(float(true_negative))/(true_negative+false_positive)
accuracy=float(true_positive+true_negative)/m
print sensitivity,specificity,accuracy
print true_positive,true_negative,false_positive,false_negative
0.0 1.0 0.525
0 21 0 19

Multiclass regression

One vs all

In [71]:
data=pd.read_csv('data4 .csv')
In [72]:
data = pd.DataFrame(data) 
data.insert(0, "constant", [1]*150, True) 
data
Out[72]:
constant f_1 f_2 f_3 f_4 f_5 f_6 f_7 output
0 1 5.1 3.5 1.4 0.2 0.04 44.275 3455.15251 1
1 1 4.9 3.0 1.4 0.2 0.04 28.400 2829.15249 1
2 1 4.7 3.2 1.3 0.2 0.04 34.068 2297.95007 1
3 1 4.6 3.1 1.5 0.2 0.04 31.291 2064.22976 1
4 1 5.0 3.6 1.4 0.2 0.04 48.056 3130.00000 1
5 1 5.4 3.9 1.7 0.4 0.16 61.019 4597.25024 1
6 1 4.6 3.4 1.4 0.3 0.09 40.704 2064.42976 1
7 1 5.0 3.4 1.5 0.2 0.04 40.804 3129.90000 1
8 1 4.4 2.9 1.4 0.2 0.04 25.789 1653.46224 1
9 1 4.9 3.1 1.5 0.1 0.01 31.291 2829.35249 1
10 1 5.4 3.7 1.5 0.2 0.04 52.153 4596.85024 1
11 1 4.8 3.4 1.6 0.2 0.04 40.904 2553.03968 1
12 1 4.8 3.0 1.4 0.1 0.01 28.400 2552.43968 1
13 1 4.3 3.0 1.1 0.1 0.01 28.100 1474.18443 1
14 1 5.8 4.0 1.2 0.2 0.04 65.200 6568.76768 1
15 1 5.7 4.4 1.5 0.4 0.16 86.684 6022.82057 1
16 1 5.4 3.9 1.3 0.4 0.16 60.619 4596.85024 1
17 1 5.1 3.5 1.4 0.3 0.09 44.275 3455.15251 1
18 1 5.7 3.8 1.7 0.3 0.09 56.572 6022.42057 1
19 1 5.1 3.8 1.5 0.3 0.09 56.372 3455.55251 1
20 1 5.4 3.4 1.7 0.2 0.04 41.004 4596.75024 1
21 1 5.1 3.7 1.5 0.4 0.16 52.153 3455.45251 1
22 1 4.6 3.6 1.0 0.2 0.04 47.656 2064.22976 1
23 1 5.1 3.3 1.7 0.5 0.25 37.637 3455.25251 1
24 1 4.8 3.4 1.9 0.2 0.04 41.204 2553.33968 1
25 1 5.0 3.0 1.6 0.2 0.04 28.600 3129.60000 1
26 1 5.0 3.4 1.6 0.4 0.16 40.904 3130.00000 1
27 1 5.2 3.5 1.5 0.2 0.04 44.375 3807.04032 1
28 1 5.2 3.4 1.4 0.2 0.04 40.704 3806.84032 1
29 1 4.7 3.2 1.6 0.2 0.04 34.368 2298.25007 1
... ... ... ... ... ... ... ... ... ...
120 1 6.9 3.2 5.7 2.3 5.29 38.468 15649.21349 3
121 1 5.6 2.8 4.9 2.0 4.00 26.852 5515.01776 3
122 1 7.7 2.8 6.7 2.0 4.00 28.652 27077.34157 3
123 1 6.3 2.7 4.9 1.8 3.24 24.583 9931.96543 3
124 1 6.7 3.3 5.7 2.1 4.41 41.637 13510.25107 3
125 1 7.2 3.2 6.0 1.8 3.24 38.768 19358.37632 3
126 1 6.2 2.8 4.8 1.8 3.24 26.752 9168.92832 3
127 1 6.1 3.0 4.9 1.8 3.24 31.900 8453.86301 3
128 1 6.4 2.8 5.6 2.1 4.41 27.552 10745.81824 3
129 1 7.2 3.0 5.8 1.6 2.56 32.800 19357.97632 3
130 1 7.4 2.8 6.1 1.9 3.61 28.052 22198.96624 3
131 1 7.9 3.8 6.4 2.0 4.00 61.272 30780.76399 3
132 1 6.4 2.8 5.6 2.2 4.84 27.552 10745.81824 3
133 1 6.3 2.8 5.1 1.5 2.25 27.052 9932.26543 3
134 1 6.1 2.6 5.6 1.4 1.96 23.176 8454.16301 3
135 1 7.7 3.0 6.1 2.3 5.29 33.100 27076.94157 3
136 1 6.3 3.4 5.6 2.4 5.76 44.904 9933.36543 3
137 1 6.4 3.1 5.5 1.8 3.24 35.291 10746.01824 3
138 1 6.0 3.0 4.8 1.8 3.24 31.800 7783.80000 3
139 1 6.9 3.1 5.4 2.1 4.41 35.191 15648.81349 3
140 1 6.7 3.1 5.6 2.4 5.76 35.391 13509.95107 3
141 1 6.9 3.1 5.1 2.3 5.29 34.891 15648.51349 3
142 1 5.8 2.7 5.1 1.9 3.61 24.783 6571.36768 3
143 1 6.8 3.2 5.9 2.3 5.29 38.668 14548.43568 3
144 1 6.7 3.3 5.7 2.5 6.25 41.637 13510.25107 3
145 1 6.7 3.0 5.2 2.3 5.29 32.200 13509.45107 3
146 1 6.3 2.5 5.0 1.9 3.61 20.625 9931.86543 3
147 1 6.5 3.0 5.2 2.0 4.00 32.200 11611.10625 3
148 1 6.2 3.4 5.4 2.3 5.29 44.704 9170.12832 3
149 1 5.9 3.0 5.1 1.8 3.24 32.100 7157.34299 3

150 rows × 9 columns

In [73]:
data
Out[73]:
constant f_1 f_2 f_3 f_4 f_5 f_6 f_7 output
0 1 5.1 3.5 1.4 0.2 0.04 44.275 3455.15251 1
1 1 4.9 3.0 1.4 0.2 0.04 28.400 2829.15249 1
2 1 4.7 3.2 1.3 0.2 0.04 34.068 2297.95007 1
3 1 4.6 3.1 1.5 0.2 0.04 31.291 2064.22976 1
4 1 5.0 3.6 1.4 0.2 0.04 48.056 3130.00000 1
5 1 5.4 3.9 1.7 0.4 0.16 61.019 4597.25024 1
6 1 4.6 3.4 1.4 0.3 0.09 40.704 2064.42976 1
7 1 5.0 3.4 1.5 0.2 0.04 40.804 3129.90000 1
8 1 4.4 2.9 1.4 0.2 0.04 25.789 1653.46224 1
9 1 4.9 3.1 1.5 0.1 0.01 31.291 2829.35249 1
10 1 5.4 3.7 1.5 0.2 0.04 52.153 4596.85024 1
11 1 4.8 3.4 1.6 0.2 0.04 40.904 2553.03968 1
12 1 4.8 3.0 1.4 0.1 0.01 28.400 2552.43968 1
13 1 4.3 3.0 1.1 0.1 0.01 28.100 1474.18443 1
14 1 5.8 4.0 1.2 0.2 0.04 65.200 6568.76768 1
15 1 5.7 4.4 1.5 0.4 0.16 86.684 6022.82057 1
16 1 5.4 3.9 1.3 0.4 0.16 60.619 4596.85024 1
17 1 5.1 3.5 1.4 0.3 0.09 44.275 3455.15251 1
18 1 5.7 3.8 1.7 0.3 0.09 56.572 6022.42057 1
19 1 5.1 3.8 1.5 0.3 0.09 56.372 3455.55251 1
20 1 5.4 3.4 1.7 0.2 0.04 41.004 4596.75024 1
21 1 5.1 3.7 1.5 0.4 0.16 52.153 3455.45251 1
22 1 4.6 3.6 1.0 0.2 0.04 47.656 2064.22976 1
23 1 5.1 3.3 1.7 0.5 0.25 37.637 3455.25251 1
24 1 4.8 3.4 1.9 0.2 0.04 41.204 2553.33968 1
25 1 5.0 3.0 1.6 0.2 0.04 28.600 3129.60000 1
26 1 5.0 3.4 1.6 0.4 0.16 40.904 3130.00000 1
27 1 5.2 3.5 1.5 0.2 0.04 44.375 3807.04032 1
28 1 5.2 3.4 1.4 0.2 0.04 40.704 3806.84032 1
29 1 4.7 3.2 1.6 0.2 0.04 34.368 2298.25007 1
... ... ... ... ... ... ... ... ... ...
120 1 6.9 3.2 5.7 2.3 5.29 38.468 15649.21349 3
121 1 5.6 2.8 4.9 2.0 4.00 26.852 5515.01776 3
122 1 7.7 2.8 6.7 2.0 4.00 28.652 27077.34157 3
123 1 6.3 2.7 4.9 1.8 3.24 24.583 9931.96543 3
124 1 6.7 3.3 5.7 2.1 4.41 41.637 13510.25107 3
125 1 7.2 3.2 6.0 1.8 3.24 38.768 19358.37632 3
126 1 6.2 2.8 4.8 1.8 3.24 26.752 9168.92832 3
127 1 6.1 3.0 4.9 1.8 3.24 31.900 8453.86301 3
128 1 6.4 2.8 5.6 2.1 4.41 27.552 10745.81824 3
129 1 7.2 3.0 5.8 1.6 2.56 32.800 19357.97632 3
130 1 7.4 2.8 6.1 1.9 3.61 28.052 22198.96624 3
131 1 7.9 3.8 6.4 2.0 4.00 61.272 30780.76399 3
132 1 6.4 2.8 5.6 2.2 4.84 27.552 10745.81824 3
133 1 6.3 2.8 5.1 1.5 2.25 27.052 9932.26543 3
134 1 6.1 2.6 5.6 1.4 1.96 23.176 8454.16301 3
135 1 7.7 3.0 6.1 2.3 5.29 33.100 27076.94157 3
136 1 6.3 3.4 5.6 2.4 5.76 44.904 9933.36543 3
137 1 6.4 3.1 5.5 1.8 3.24 35.291 10746.01824 3
138 1 6.0 3.0 4.8 1.8 3.24 31.800 7783.80000 3
139 1 6.9 3.1 5.4 2.1 4.41 35.191 15648.81349 3
140 1 6.7 3.1 5.6 2.4 5.76 35.391 13509.95107 3
141 1 6.9 3.1 5.1 2.3 5.29 34.891 15648.51349 3
142 1 5.8 2.7 5.1 1.9 3.61 24.783 6571.36768 3
143 1 6.8 3.2 5.9 2.3 5.29 38.668 14548.43568 3
144 1 6.7 3.3 5.7 2.5 6.25 41.637 13510.25107 3
145 1 6.7 3.0 5.2 2.3 5.29 32.200 13509.45107 3
146 1 6.3 2.5 5.0 1.9 3.61 20.625 9931.86543 3
147 1 6.5 3.0 5.2 2.0 4.00 32.200 11611.10625 3
148 1 6.2 3.4 5.4 2.3 5.29 44.704 9170.12832 3
149 1 5.9 3.0 5.1 1.8 3.24 32.100 7157.34299 3

150 rows × 9 columns

Output 1 vs 2 , 3

In [74]:
def sigmoid(x):
    y=(1/(1+np.exp(-x)))
    return y
In [75]:
train=data.sample(frac=0.6,random_state=200) #random state is a seed value
test=data.drop(train.index)

train
Out[75]:
constant f_1 f_2 f_3 f_4 f_5 f_6 f_7 output
84 1 5.4 3.0 4.5 1.5 2.25 31.500 4599.15024 2
122 1 7.7 2.8 6.7 2.0 4.00 28.652 27077.34157 3
28 1 5.2 3.4 1.4 0.2 0.04 40.704 3806.84032 1
24 1 4.8 3.4 1.9 0.2 0.04 41.204 2553.33968 1
75 1 6.6 3.0 4.4 1.4 1.96 31.400 12530.72576 2
109 1 7.2 3.6 6.1 2.5 6.25 52.756 19358.87632 3
81 1 5.5 2.4 3.7 1.0 1.00 17.524 5038.94375 2
98 1 5.1 2.5 3.0 1.1 1.21 18.625 3455.75251 2
80 1 5.5 2.4 3.8 1.1 1.21 17.624 5039.04375 2
100 1 6.3 3.3 6.0 2.5 6.25 41.937 9933.66543 3
124 1 6.7 3.3 5.7 2.1 4.41 41.637 13510.25107 3
2 1 4.7 3.2 1.3 0.2 0.04 34.068 2297.95007 1
34 1 4.9 3.1 1.5 0.1 0.01 31.291 2829.35249 1
44 1 5.1 3.8 1.9 0.4 0.16 56.772 3455.95251 1
128 1 6.4 2.8 5.6 2.1 4.41 27.552 10745.81824 3
13 1 4.3 3.0 1.1 0.1 0.01 28.100 1474.18443 1
93 1 5.0 2.3 3.3 1.0 1.00 15.467 3130.60000 2
41 1 4.5 2.3 1.3 0.3 0.09 13.467 1848.88125 1
63 1 6.1 2.9 4.7 1.4 1.96 29.089 8453.56301 2
137 1 6.4 3.1 5.5 1.8 3.24 35.291 10746.01824 3
19 1 5.1 3.8 1.5 0.3 0.09 56.372 3455.55251 1
60 1 5.0 2.0 3.5 1.0 1.00 11.500 3130.50000 2
126 1 6.2 2.8 4.8 1.8 3.24 26.752 9168.92832 3
11 1 4.8 3.4 1.6 0.2 0.04 40.904 2553.03968 1
47 1 4.6 3.2 1.4 0.2 0.04 34.168 2064.22976 1
18 1 5.7 3.8 1.7 0.3 0.09 56.572 6022.42057 1
96 1 5.7 2.9 4.2 1.3 1.69 28.589 6024.02057 2
107 1 7.3 2.9 6.3 1.8 3.24 30.689 20739.91593 3
141 1 6.9 3.1 5.1 2.3 5.29 34.891 15648.51349 3
53 1 5.5 2.3 4.0 1.3 1.69 16.167 5039.14375 2
... ... ... ... ... ... ... ... ... ...
8 1 4.4 2.9 1.4 0.2 0.04 25.789 1653.46224 1
49 1 5.0 3.3 1.4 0.2 0.04 37.337 3129.70000 1
112 1 6.8 3.0 5.5 2.1 4.41 32.500 14547.83568 3
17 1 5.1 3.5 1.4 0.3 0.09 44.275 3455.15251 1
15 1 5.7 4.4 1.5 0.4 0.16 86.684 6022.82057 1
104 1 6.5 3.0 5.8 2.2 4.84 32.800 11611.70625 3
91 1 6.1 3.0 4.6 1.4 1.96 31.600 8453.56301 2
46 1 5.1 3.8 1.6 0.2 0.04 56.472 3455.65251 1
43 1 5.0 3.5 1.6 0.6 0.36 44.475 3130.10000 1
101 1 5.8 2.7 5.1 1.9 3.61 24.783 6571.36768 3
37 1 4.9 3.1 1.5 0.1 0.01 31.291 2829.35249 1
38 1 4.4 3.0 1.3 0.2 0.04 28.300 1653.46224 1
125 1 7.2 3.2 6.0 1.8 3.24 38.768 19358.37632 3
103 1 6.3 2.9 5.6 1.8 3.24 29.989 9932.86543 3
71 1 6.1 2.8 4.0 1.3 1.69 25.952 8452.76301 2
65 1 6.7 3.1 4.4 1.4 1.96 34.191 13508.75107 2
149 1 5.9 3.0 5.1 1.8 3.24 32.100 7157.34299 3
40 1 5.0 3.5 1.3 0.3 0.09 44.175 3129.80000 1
5 1 5.4 3.9 1.7 0.4 0.16 61.019 4597.25024 1
131 1 7.9 3.8 6.4 2.0 4.00 61.272 30780.76399 3
57 1 4.9 2.4 3.3 1.0 1.00 17.124 2830.45249 2
30 1 4.8 3.1 1.6 0.2 0.04 31.391 2552.73968 1
148 1 6.2 3.4 5.4 2.3 5.29 44.704 9170.12832 3
132 1 6.4 2.8 5.6 2.2 4.84 27.552 10745.81824 3
22 1 4.6 3.6 1.0 0.2 0.04 47.656 2064.22976 1
29 1 4.7 3.2 1.6 0.2 0.04 34.368 2298.25007 1
50 1 7.0 3.2 4.7 1.4 1.96 37.468 16814.90000 2
140 1 6.7 3.1 5.6 2.4 5.76 35.391 13509.95107 3
64 1 5.6 2.9 3.6 1.3 1.69 27.989 5513.81776 2
108 1 6.7 2.5 5.8 1.8 3.24 21.425 13509.55107 3

90 rows × 9 columns

In [76]:
train['output'] = (train['output'] < 2)*1

train
Out[76]:
constant f_1 f_2 f_3 f_4 f_5 f_6 f_7 output
84 1 5.4 3.0 4.5 1.5 2.25 31.500 4599.15024 0
122 1 7.7 2.8 6.7 2.0 4.00 28.652 27077.34157 0
28 1 5.2 3.4 1.4 0.2 0.04 40.704 3806.84032 1
24 1 4.8 3.4 1.9 0.2 0.04 41.204 2553.33968 1
75 1 6.6 3.0 4.4 1.4 1.96 31.400 12530.72576 0
109 1 7.2 3.6 6.1 2.5 6.25 52.756 19358.87632 0
81 1 5.5 2.4 3.7 1.0 1.00 17.524 5038.94375 0
98 1 5.1 2.5 3.0 1.1 1.21 18.625 3455.75251 0
80 1 5.5 2.4 3.8 1.1 1.21 17.624 5039.04375 0
100 1 6.3 3.3 6.0 2.5 6.25 41.937 9933.66543 0
124 1 6.7 3.3 5.7 2.1 4.41 41.637 13510.25107 0
2 1 4.7 3.2 1.3 0.2 0.04 34.068 2297.95007 1
34 1 4.9 3.1 1.5 0.1 0.01 31.291 2829.35249 1
44 1 5.1 3.8 1.9 0.4 0.16 56.772 3455.95251 1
128 1 6.4 2.8 5.6 2.1 4.41 27.552 10745.81824 0
13 1 4.3 3.0 1.1 0.1 0.01 28.100 1474.18443 1
93 1 5.0 2.3 3.3 1.0 1.00 15.467 3130.60000 0
41 1 4.5 2.3 1.3 0.3 0.09 13.467 1848.88125 1
63 1 6.1 2.9 4.7 1.4 1.96 29.089 8453.56301 0
137 1 6.4 3.1 5.5 1.8 3.24 35.291 10746.01824 0
19 1 5.1 3.8 1.5 0.3 0.09 56.372 3455.55251 1
60 1 5.0 2.0 3.5 1.0 1.00 11.500 3130.50000 0
126 1 6.2 2.8 4.8 1.8 3.24 26.752 9168.92832 0
11 1 4.8 3.4 1.6 0.2 0.04 40.904 2553.03968 1
47 1 4.6 3.2 1.4 0.2 0.04 34.168 2064.22976 1
18 1 5.7 3.8 1.7 0.3 0.09 56.572 6022.42057 1
96 1 5.7 2.9 4.2 1.3 1.69 28.589 6024.02057 0
107 1 7.3 2.9 6.3 1.8 3.24 30.689 20739.91593 0
141 1 6.9 3.1 5.1 2.3 5.29 34.891 15648.51349 0
53 1 5.5 2.3 4.0 1.3 1.69 16.167 5039.14375 0
... ... ... ... ... ... ... ... ... ...
8 1 4.4 2.9 1.4 0.2 0.04 25.789 1653.46224 1
49 1 5.0 3.3 1.4 0.2 0.04 37.337 3129.70000 1
112 1 6.8 3.0 5.5 2.1 4.41 32.500 14547.83568 0
17 1 5.1 3.5 1.4 0.3 0.09 44.275 3455.15251 1
15 1 5.7 4.4 1.5 0.4 0.16 86.684 6022.82057 1
104 1 6.5 3.0 5.8 2.2 4.84 32.800 11611.70625 0
91 1 6.1 3.0 4.6 1.4 1.96 31.600 8453.56301 0
46 1 5.1 3.8 1.6 0.2 0.04 56.472 3455.65251 1
43 1 5.0 3.5 1.6 0.6 0.36 44.475 3130.10000 1
101 1 5.8 2.7 5.1 1.9 3.61 24.783 6571.36768 0
37 1 4.9 3.1 1.5 0.1 0.01 31.291 2829.35249 1
38 1 4.4 3.0 1.3 0.2 0.04 28.300 1653.46224 1
125 1 7.2 3.2 6.0 1.8 3.24 38.768 19358.37632 0
103 1 6.3 2.9 5.6 1.8 3.24 29.989 9932.86543 0
71 1 6.1 2.8 4.0 1.3 1.69 25.952 8452.76301 0
65 1 6.7 3.1 4.4 1.4 1.96 34.191 13508.75107 0
149 1 5.9 3.0 5.1 1.8 3.24 32.100 7157.34299 0
40 1 5.0 3.5 1.3 0.3 0.09 44.175 3129.80000 1
5 1 5.4 3.9 1.7 0.4 0.16 61.019 4597.25024 1
131 1 7.9 3.8 6.4 2.0 4.00 61.272 30780.76399 0
57 1 4.9 2.4 3.3 1.0 1.00 17.124 2830.45249 0
30 1 4.8 3.1 1.6 0.2 0.04 31.391 2552.73968 1
148 1 6.2 3.4 5.4 2.3 5.29 44.704 9170.12832 0
132 1 6.4 2.8 5.6 2.2 4.84 27.552 10745.81824 0
22 1 4.6 3.6 1.0 0.2 0.04 47.656 2064.22976 1
29 1 4.7 3.2 1.6 0.2 0.04 34.368 2298.25007 1
50 1 7.0 3.2 4.7 1.4 1.96 37.468 16814.90000 0
140 1 6.7 3.1 5.6 2.4 5.76 35.391 13509.95107 0
64 1 5.6 2.9 3.6 1.3 1.69 27.989 5513.81776 0
108 1 6.7 2.5 5.8 1.8 3.24 21.425 13509.55107 0

90 rows × 9 columns

In [77]:
train=train.to_numpy()
In [78]:
train[:,6:8]=feature_normalize(train[:,6:8])
In [79]:
w=np.ones(8)
(weight,cost)=lgd(.1,train[:,0:8],train[:,8],w,250)
print weight,cost
[ 1.01898376  0.74961263  1.14674712 -0.49146472  0.18539606 -0.12893102
  0.60735951 -0.35990046] [1.11449987 0.97483258 0.86800399 0.78455894 0.7188427  0.66693832
 0.62588192 0.59334728 0.56749551 0.54687962 0.53036911 0.51708555
 0.50634819 0.49762939 0.49051913 0.48469754 0.47991372 0.47596974
 0.47270855 0.47000474 0.46775765 0.46588602 0.46432394 0.46301773
 0.46192347 0.46100511 0.46023301 0.45958266 0.45903381 0.45856966
 0.45817629 0.45784208 0.45755739 0.45731417 0.45710572 0.45692641
 0.45677158 0.45663731 0.45652033 0.45641789 0.4563277  0.45624784
 0.45617671 0.45611294 0.45605542 0.45600318 0.45595544 0.45591153
 0.45587088 0.45583303 0.45579758 0.45576419 0.45573258 0.45570252
 0.4556738  0.45564626 0.45561976 0.45559418 0.4555694  0.45554536
 0.45552196 0.45549916 0.45547689 0.45545511 0.45543378 0.45541286
 0.45539233 0.45537217 0.45535233 0.45533282 0.45531361 0.45529468
 0.45527602 0.45525763 0.45523948 0.45522157 0.45520388 0.45518642
 0.45516917 0.45515212 0.45513527 0.45511862 0.45510215 0.45508586
 0.45506975 0.45505381 0.45503803 0.45502242 0.45500696 0.45499166
 0.4549765  0.45496149 0.45494662 0.45493189 0.4549173  0.45490283
 0.4548885  0.45487429 0.4548602  0.45484623 0.45483237 0.45481864
 0.45480501 0.45479149 0.45477807 0.45476476 0.45475155 0.45473844
 0.45472543 0.45471251 0.45469968 0.45468694 0.45467429 0.45466173
 0.45464925 0.45463686 0.45462455 0.45461231 0.45460016 0.45458808
 0.45457607 0.45456414 0.45455228 0.45454049 0.45452876 0.45451711
 0.45450552 0.454494   0.45448254 0.45447114 0.45445981 0.45444853
 0.45443732 0.45442616 0.45441506 0.45440401 0.45439302 0.45438209
 0.4543712  0.45436037 0.4543496  0.45433887 0.45432819 0.45431756
 0.45430698 0.45429645 0.45428597 0.45427553 0.45426513 0.45425478
 0.45424448 0.45423422 0.454224   0.45421383 0.4542037  0.45419361
 0.45418356 0.45417355 0.45416358 0.45415365 0.45414376 0.45413391
 0.4541241  0.45411432 0.45410459 0.45409489 0.45408522 0.4540756
 0.45406601 0.45405645 0.45404693 0.45403745 0.454028   0.45401859
 0.4540092  0.45399986 0.45399055 0.45398127 0.45397202 0.45396281
 0.45395363 0.45394448 0.45393537 0.45392628 0.45391723 0.45390821
 0.45389922 0.45389027 0.45388134 0.45387245 0.45386359 0.45385475
 0.45384595 0.45383718 0.45382844 0.45381973 0.45381105 0.4538024
 0.45379378 0.45378518 0.45377662 0.45376809 0.45375959 0.45375111
 0.45374267 0.45373425 0.45372586 0.45371751 0.45370918 0.45370087
 0.4536926  0.45368436 0.45367614 0.45366796 0.4536598  0.45365167
 0.45364356 0.45363549 0.45362744 0.45361942 0.45361143 0.45360347
 0.45359553 0.45358762 0.45357974 0.45357189 0.45356406 0.45355626
 0.45354849 0.45354075 0.45353303 0.45352534 0.45351768 0.45351005
 0.45350244 0.45349486 0.4534873  0.45347978 0.45347228 0.4534648
 0.45345735 0.45344993 0.45344254 0.45343517 0.45342783 0.45342052
 0.45341323 0.45340597 0.45339874 0.45339153]
In [80]:
train
train[:,6:8]=feature_normalize(train[:,6:8])
In [81]:
h_hat=np.zeros(90)
for i in range(90):
    for j in range(8):
        h_hat[i]+=weight[j]*train[i][j]
    
fig=plt.figure(figsize=(9,8))
ax=fig.add_subplot(111)
colors = ['#483d8b', '#cc8400']
colors_data = [colors[int(i)] for i in train[:,8]]
ax.scatter(train[:,1],train[:,2], color=colors_data, s=100, edgecolor='black', linewidth=2, alpha=0.7)
ax.plot(h_hat, linewidth=2, color='#483d8b')
plt.xlim([4,8])
plt.ylim([1.5,3.9])
Out[81]:
(1.5, 3.9)

3 vs 1, 2

In [82]:
data=pd.read_csv('data4 .csv')
data = pd.DataFrame(data) 
data.insert(0, "constant", [1]*150, True) 
train=data.sample(frac=0.6,random_state=200) #random state is a seed value
test=data.drop(train.index)
train['output'] = (train['output'] >=3) * 1
train
Out[82]:
constant f_1 f_2 f_3 f_4 f_5 f_6 f_7 output
84 1 5.4 3.0 4.5 1.5 2.25 31.500 4599.15024 0
122 1 7.7 2.8 6.7 2.0 4.00 28.652 27077.34157 1
28 1 5.2 3.4 1.4 0.2 0.04 40.704 3806.84032 0
24 1 4.8 3.4 1.9 0.2 0.04 41.204 2553.33968 0
75 1 6.6 3.0 4.4 1.4 1.96 31.400 12530.72576 0
109 1 7.2 3.6 6.1 2.5 6.25 52.756 19358.87632 1
81 1 5.5 2.4 3.7 1.0 1.00 17.524 5038.94375 0
98 1 5.1 2.5 3.0 1.1 1.21 18.625 3455.75251 0
80 1 5.5 2.4 3.8 1.1 1.21 17.624 5039.04375 0
100 1 6.3 3.3 6.0 2.5 6.25 41.937 9933.66543 1
124 1 6.7 3.3 5.7 2.1 4.41 41.637 13510.25107 1
2 1 4.7 3.2 1.3 0.2 0.04 34.068 2297.95007 0
34 1 4.9 3.1 1.5 0.1 0.01 31.291 2829.35249 0
44 1 5.1 3.8 1.9 0.4 0.16 56.772 3455.95251 0
128 1 6.4 2.8 5.6 2.1 4.41 27.552 10745.81824 1
13 1 4.3 3.0 1.1 0.1 0.01 28.100 1474.18443 0
93 1 5.0 2.3 3.3 1.0 1.00 15.467 3130.60000 0
41 1 4.5 2.3 1.3 0.3 0.09 13.467 1848.88125 0
63 1 6.1 2.9 4.7 1.4 1.96 29.089 8453.56301 0
137 1 6.4 3.1 5.5 1.8 3.24 35.291 10746.01824 1
19 1 5.1 3.8 1.5 0.3 0.09 56.372 3455.55251 0
60 1 5.0 2.0 3.5 1.0 1.00 11.500 3130.50000 0
126 1 6.2 2.8 4.8 1.8 3.24 26.752 9168.92832 1
11 1 4.8 3.4 1.6 0.2 0.04 40.904 2553.03968 0
47 1 4.6 3.2 1.4 0.2 0.04 34.168 2064.22976 0
18 1 5.7 3.8 1.7 0.3 0.09 56.572 6022.42057 0
96 1 5.7 2.9 4.2 1.3 1.69 28.589 6024.02057 0
107 1 7.3 2.9 6.3 1.8 3.24 30.689 20739.91593 1
141 1 6.9 3.1 5.1 2.3 5.29 34.891 15648.51349 1
53 1 5.5 2.3 4.0 1.3 1.69 16.167 5039.14375 0
... ... ... ... ... ... ... ... ... ...
8 1 4.4 2.9 1.4 0.2 0.04 25.789 1653.46224 0
49 1 5.0 3.3 1.4 0.2 0.04 37.337 3129.70000 0
112 1 6.8 3.0 5.5 2.1 4.41 32.500 14547.83568 1
17 1 5.1 3.5 1.4 0.3 0.09 44.275 3455.15251 0
15 1 5.7 4.4 1.5 0.4 0.16 86.684 6022.82057 0
104 1 6.5 3.0 5.8 2.2 4.84 32.800 11611.70625 1
91 1 6.1 3.0 4.6 1.4 1.96 31.600 8453.56301 0
46 1 5.1 3.8 1.6 0.2 0.04 56.472 3455.65251 0
43 1 5.0 3.5 1.6 0.6 0.36 44.475 3130.10000 0
101 1 5.8 2.7 5.1 1.9 3.61 24.783 6571.36768 1
37 1 4.9 3.1 1.5 0.1 0.01 31.291 2829.35249 0
38 1 4.4 3.0 1.3 0.2 0.04 28.300 1653.46224 0
125 1 7.2 3.2 6.0 1.8 3.24 38.768 19358.37632 1
103 1 6.3 2.9 5.6 1.8 3.24 29.989 9932.86543 1
71 1 6.1 2.8 4.0 1.3 1.69 25.952 8452.76301 0
65 1 6.7 3.1 4.4 1.4 1.96 34.191 13508.75107 0
149 1 5.9 3.0 5.1 1.8 3.24 32.100 7157.34299 1
40 1 5.0 3.5 1.3 0.3 0.09 44.175 3129.80000 0
5 1 5.4 3.9 1.7 0.4 0.16 61.019 4597.25024 0
131 1 7.9 3.8 6.4 2.0 4.00 61.272 30780.76399 1
57 1 4.9 2.4 3.3 1.0 1.00 17.124 2830.45249 0
30 1 4.8 3.1 1.6 0.2 0.04 31.391 2552.73968 0
148 1 6.2 3.4 5.4 2.3 5.29 44.704 9170.12832 1
132 1 6.4 2.8 5.6 2.2 4.84 27.552 10745.81824 1
22 1 4.6 3.6 1.0 0.2 0.04 47.656 2064.22976 0
29 1 4.7 3.2 1.6 0.2 0.04 34.368 2298.25007 0
50 1 7.0 3.2 4.7 1.4 1.96 37.468 16814.90000 0
140 1 6.7 3.1 5.6 2.4 5.76 35.391 13509.95107 1
64 1 5.6 2.9 3.6 1.3 1.69 27.989 5513.81776 0
108 1 6.7 2.5 5.8 1.8 3.24 21.425 13509.55107 1

90 rows × 9 columns

In [83]:
train=train.to_numpy()
In [ ]:
 
In [85]:
w=np.ones(8)
(weight,cost)=lgd(0.1,train[:,0:8],train[:,8],w,300)
print weight
[0.05114464 0.0562259  0.05366255 0.05398383 0.05137772 0.05285967
 0.08869747 0.19294561]
In [86]:
h_hat=np.zeros(90)
for i in range(90):
    for j in range(8):
        h_hat[i]+=weight[j]*train[i][j]
    
fig=plt.figure(figsize=(8, 4))
ax=fig.add_subplot(111)
colors = ['#483d8b', '#cc8400','red','green']
colors_data = [colors[int(i)] for i in train[:,8]]
ax.scatter(train[:,1],train[:,2], color=colors_data, s=100, edgecolor='black', linewidth=2, alpha=0.7)
ax.plot(h_hat, linewidth=2, color='#483d8b')
plt.xlim([4,8])
plt.ylim([1.7, 4])
Out[86]:
(1.7, 4)

2 vs 1, 3

In [87]:
data=pd.read_csv('data4 .csv')
data = pd.DataFrame(data) 
data.insert(0, "constant", [1]*150, True) 
train=data.sample(frac=0.6,random_state=200) #random state is a seed value
test=data.drop(train.index)
train['output'] = (train['output'] == 2) * 1
train
Out[87]:
constant f_1 f_2 f_3 f_4 f_5 f_6 f_7 output
84 1 5.4 3.0 4.5 1.5 2.25 31.500 4599.15024 1
122 1 7.7 2.8 6.7 2.0 4.00 28.652 27077.34157 0
28 1 5.2 3.4 1.4 0.2 0.04 40.704 3806.84032 0
24 1 4.8 3.4 1.9 0.2 0.04 41.204 2553.33968 0
75 1 6.6 3.0 4.4 1.4 1.96 31.400 12530.72576 1
109 1 7.2 3.6 6.1 2.5 6.25 52.756 19358.87632 0
81 1 5.5 2.4 3.7 1.0 1.00 17.524 5038.94375 1
98 1 5.1 2.5 3.0 1.1 1.21 18.625 3455.75251 1
80 1 5.5 2.4 3.8 1.1 1.21 17.624 5039.04375 1
100 1 6.3 3.3 6.0 2.5 6.25 41.937 9933.66543 0
124 1 6.7 3.3 5.7 2.1 4.41 41.637 13510.25107 0
2 1 4.7 3.2 1.3 0.2 0.04 34.068 2297.95007 0
34 1 4.9 3.1 1.5 0.1 0.01 31.291 2829.35249 0
44 1 5.1 3.8 1.9 0.4 0.16 56.772 3455.95251 0
128 1 6.4 2.8 5.6 2.1 4.41 27.552 10745.81824 0
13 1 4.3 3.0 1.1 0.1 0.01 28.100 1474.18443 0
93 1 5.0 2.3 3.3 1.0 1.00 15.467 3130.60000 1
41 1 4.5 2.3 1.3 0.3 0.09 13.467 1848.88125 0
63 1 6.1 2.9 4.7 1.4 1.96 29.089 8453.56301 1
137 1 6.4 3.1 5.5 1.8 3.24 35.291 10746.01824 0
19 1 5.1 3.8 1.5 0.3 0.09 56.372 3455.55251 0
60 1 5.0 2.0 3.5 1.0 1.00 11.500 3130.50000 1
126 1 6.2 2.8 4.8 1.8 3.24 26.752 9168.92832 0
11 1 4.8 3.4 1.6 0.2 0.04 40.904 2553.03968 0
47 1 4.6 3.2 1.4 0.2 0.04 34.168 2064.22976 0
18 1 5.7 3.8 1.7 0.3 0.09 56.572 6022.42057 0
96 1 5.7 2.9 4.2 1.3 1.69 28.589 6024.02057 1
107 1 7.3 2.9 6.3 1.8 3.24 30.689 20739.91593 0
141 1 6.9 3.1 5.1 2.3 5.29 34.891 15648.51349 0
53 1 5.5 2.3 4.0 1.3 1.69 16.167 5039.14375 1
... ... ... ... ... ... ... ... ... ...
8 1 4.4 2.9 1.4 0.2 0.04 25.789 1653.46224 0
49 1 5.0 3.3 1.4 0.2 0.04 37.337 3129.70000 0
112 1 6.8 3.0 5.5 2.1 4.41 32.500 14547.83568 0
17 1 5.1 3.5 1.4 0.3 0.09 44.275 3455.15251 0
15 1 5.7 4.4 1.5 0.4 0.16 86.684 6022.82057 0
104 1 6.5 3.0 5.8 2.2 4.84 32.800 11611.70625 0
91 1 6.1 3.0 4.6 1.4 1.96 31.600 8453.56301 1
46 1 5.1 3.8 1.6 0.2 0.04 56.472 3455.65251 0
43 1 5.0 3.5 1.6 0.6 0.36 44.475 3130.10000 0
101 1 5.8 2.7 5.1 1.9 3.61 24.783 6571.36768 0
37 1 4.9 3.1 1.5 0.1 0.01 31.291 2829.35249 0
38 1 4.4 3.0 1.3 0.2 0.04 28.300 1653.46224 0
125 1 7.2 3.2 6.0 1.8 3.24 38.768 19358.37632 0
103 1 6.3 2.9 5.6 1.8 3.24 29.989 9932.86543 0
71 1 6.1 2.8 4.0 1.3 1.69 25.952 8452.76301 1
65 1 6.7 3.1 4.4 1.4 1.96 34.191 13508.75107 1
149 1 5.9 3.0 5.1 1.8 3.24 32.100 7157.34299 0
40 1 5.0 3.5 1.3 0.3 0.09 44.175 3129.80000 0
5 1 5.4 3.9 1.7 0.4 0.16 61.019 4597.25024 0
131 1 7.9 3.8 6.4 2.0 4.00 61.272 30780.76399 0
57 1 4.9 2.4 3.3 1.0 1.00 17.124 2830.45249 1
30 1 4.8 3.1 1.6 0.2 0.04 31.391 2552.73968 0
148 1 6.2 3.4 5.4 2.3 5.29 44.704 9170.12832 0
132 1 6.4 2.8 5.6 2.2 4.84 27.552 10745.81824 0
22 1 4.6 3.6 1.0 0.2 0.04 47.656 2064.22976 0
29 1 4.7 3.2 1.6 0.2 0.04 34.368 2298.25007 0
50 1 7.0 3.2 4.7 1.4 1.96 37.468 16814.90000 1
140 1 6.7 3.1 5.6 2.4 5.76 35.391 13509.95107 0
64 1 5.6 2.9 3.6 1.3 1.69 27.989 5513.81776 1
108 1 6.7 2.5 5.8 1.8 3.24 21.425 13509.55107 0

90 rows × 9 columns

In [88]:
train=train.to_numpy()
In [89]:
train[:,6:8]=feature_normalize(train[:,6:8])
In [90]:
w=np.ones(8)
(weight,cost)=lgd(0.1,train[:,0:8],train[:,8],w,250)
print weight
[ 1.11309161  0.65358926  0.27029808  0.68066613  0.74529347 -0.87154425
  0.84683075 -0.66318167]
In [91]:
h_hat=np.zeros(90)
for i in range(90):
    for j in range(8):
        h_hat[i]+=weight[j]*train[i][j]
    
fig=plt.figure(figsize=(8, 4))
ax=fig.add_subplot(111)
colors = ['#483d8b', '#cc8400','red','green']
colors_data = [colors[int(i)] for i in train[:,8]]
ax.scatter(train[:,1],train[:,2], color=colors_data, s=100, edgecolor='black', linewidth=2, alpha=0.7)
ax.plot(h_hat, linewidth=2, color='#483d8b')
plt.xlim([4, 7.5])
plt.ylim([1, 3.9])
Out[91]:
(1, 3.9)
In [92]:
test=test.to_numpy()
In [97]:
weight=[0.05114464, 0.0562259,  0.05366255, 0.05398383, 0.05137772, 0.05285967,
 0.08869747, 0.19294561]
def predict(feature,weight):
    
    m=test.shape[0]
    n=(test.shape[1])-1
    predicted=np.zeros(m)
    y=np.zeros(m)
    
    for i in range(m):
        h=0
        for j in range(n):
            h+=weight[j]*feature[i][j]
        predicted[i]=sigmoid(h)
        if(predicted[i]>=0.5):
            y[i]=1
        else:
            y[i]=0
        
    return y
    
y=(predict(test[:,0:8], weight))
print y
[1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.
 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.
 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.]
In [98]:
u1=0
u2=0
u3=0
for i in range(40):
    if(test[i,5]==y[i] and test[i,5]==1):
        u1+=1
    if(test[i,5]==y[i] and test[i,5]==2):
        u2+=1
    if(test[i,5]==y[i] and test[i,5]==3):
        u3+=1  
individual_acc_1=float(u1)/(u1+u2+u3)
individual_acc_2=float(u2)/(u1+u2+u3)
individual_acc_3=float(u3)/(u1+u2+u3)
overall_accuracy=float(u1+u2+u3)/40
print individual_acc_1,individual_acc_2,individual_acc_3
print overall_accuracy
1.0 0.0 0.0
0.05

One vs One

1 vs 2

In [99]:
data=pd.read_csv('data4 .csv')
data = pd.DataFrame(data) 
data.insert(0, "constant", [1]*150, True) 
train=data.sample(frac=0.6,random_state=200) #random state is a seed value
test=data.drop(train.index)
train[train['output']==3]=0
train['output'] = (train['output'] >=2) * 1
train['output']
train
Out[99]:
constant f_1 f_2 f_3 f_4 f_5 f_6 f_7 output
84 1 5.4 3.0 4.5 1.5 2.25 31.500 4599.15024 1
122 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
28 1 5.2 3.4 1.4 0.2 0.04 40.704 3806.84032 0
24 1 4.8 3.4 1.9 0.2 0.04 41.204 2553.33968 0
75 1 6.6 3.0 4.4 1.4 1.96 31.400 12530.72576 1
109 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
81 1 5.5 2.4 3.7 1.0 1.00 17.524 5038.94375 1
98 1 5.1 2.5 3.0 1.1 1.21 18.625 3455.75251 1
80 1 5.5 2.4 3.8 1.1 1.21 17.624 5039.04375 1
100 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
124 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
2 1 4.7 3.2 1.3 0.2 0.04 34.068 2297.95007 0
34 1 4.9 3.1 1.5 0.1 0.01 31.291 2829.35249 0
44 1 5.1 3.8 1.9 0.4 0.16 56.772 3455.95251 0
128 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
13 1 4.3 3.0 1.1 0.1 0.01 28.100 1474.18443 0
93 1 5.0 2.3 3.3 1.0 1.00 15.467 3130.60000 1
41 1 4.5 2.3 1.3 0.3 0.09 13.467 1848.88125 0
63 1 6.1 2.9 4.7 1.4 1.96 29.089 8453.56301 1
137 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
19 1 5.1 3.8 1.5 0.3 0.09 56.372 3455.55251 0
60 1 5.0 2.0 3.5 1.0 1.00 11.500 3130.50000 1
126 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
11 1 4.8 3.4 1.6 0.2 0.04 40.904 2553.03968 0
47 1 4.6 3.2 1.4 0.2 0.04 34.168 2064.22976 0
18 1 5.7 3.8 1.7 0.3 0.09 56.572 6022.42057 0
96 1 5.7 2.9 4.2 1.3 1.69 28.589 6024.02057 1
107 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
141 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
53 1 5.5 2.3 4.0 1.3 1.69 16.167 5039.14375 1
... ... ... ... ... ... ... ... ... ...
8 1 4.4 2.9 1.4 0.2 0.04 25.789 1653.46224 0
49 1 5.0 3.3 1.4 0.2 0.04 37.337 3129.70000 0
112 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
17 1 5.1 3.5 1.4 0.3 0.09 44.275 3455.15251 0
15 1 5.7 4.4 1.5 0.4 0.16 86.684 6022.82057 0
104 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
91 1 6.1 3.0 4.6 1.4 1.96 31.600 8453.56301 1
46 1 5.1 3.8 1.6 0.2 0.04 56.472 3455.65251 0
43 1 5.0 3.5 1.6 0.6 0.36 44.475 3130.10000 0
101 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
37 1 4.9 3.1 1.5 0.1 0.01 31.291 2829.35249 0
38 1 4.4 3.0 1.3 0.2 0.04 28.300 1653.46224 0
125 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
103 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
71 1 6.1 2.8 4.0 1.3 1.69 25.952 8452.76301 1
65 1 6.7 3.1 4.4 1.4 1.96 34.191 13508.75107 1
149 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
40 1 5.0 3.5 1.3 0.3 0.09 44.175 3129.80000 0
5 1 5.4 3.9 1.7 0.4 0.16 61.019 4597.25024 0
131 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
57 1 4.9 2.4 3.3 1.0 1.00 17.124 2830.45249 1
30 1 4.8 3.1 1.6 0.2 0.04 31.391 2552.73968 0
148 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
132 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
22 1 4.6 3.6 1.0 0.2 0.04 47.656 2064.22976 0
29 1 4.7 3.2 1.6 0.2 0.04 34.368 2298.25007 0
50 1 7.0 3.2 4.7 1.4 1.96 37.468 16814.90000 1
140 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
64 1 5.6 2.9 3.6 1.3 1.69 27.989 5513.81776 1
108 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0

90 rows × 9 columns

In [100]:
train=train.to_numpy()
In [101]:
w=np.ones(8)

(weight,cost)=lgd(0.1,train[:,0:8],train[:,8],w,300)
print weight
[0.02939423 0.03329167 0.02993446 0.03354443 0.03008265 0.03049143
 0.0199265  0.18541571]

J=1.08252153

2 vs 3

In [102]:
data=pd.read_csv('data4 .csv')
data = pd.DataFrame(data) 
data.insert(0, "constant", [1]*150, True) 
train=data.sample(frac=0.6,random_state=200) #random state is a seed value
test=data.drop(train.index)
train[train['output']==1]=0
train['output'] = (train['output']==3) * 1
train['output']
train
Out[102]:
constant f_1 f_2 f_3 f_4 f_5 f_6 f_7 output
84 1 5.4 3.0 4.5 1.5 2.25 31.500 4599.15024 0
122 1 7.7 2.8 6.7 2.0 4.00 28.652 27077.34157 1
28 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
24 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
75 1 6.6 3.0 4.4 1.4 1.96 31.400 12530.72576 0
109 1 7.2 3.6 6.1 2.5 6.25 52.756 19358.87632 1
81 1 5.5 2.4 3.7 1.0 1.00 17.524 5038.94375 0
98 1 5.1 2.5 3.0 1.1 1.21 18.625 3455.75251 0
80 1 5.5 2.4 3.8 1.1 1.21 17.624 5039.04375 0
100 1 6.3 3.3 6.0 2.5 6.25 41.937 9933.66543 1
124 1 6.7 3.3 5.7 2.1 4.41 41.637 13510.25107 1
2 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
34 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
44 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
128 1 6.4 2.8 5.6 2.1 4.41 27.552 10745.81824 1
13 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
93 1 5.0 2.3 3.3 1.0 1.00 15.467 3130.60000 0
41 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
63 1 6.1 2.9 4.7 1.4 1.96 29.089 8453.56301 0
137 1 6.4 3.1 5.5 1.8 3.24 35.291 10746.01824 1
19 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
60 1 5.0 2.0 3.5 1.0 1.00 11.500 3130.50000 0
126 1 6.2 2.8 4.8 1.8 3.24 26.752 9168.92832 1
11 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
47 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
18 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
96 1 5.7 2.9 4.2 1.3 1.69 28.589 6024.02057 0
107 1 7.3 2.9 6.3 1.8 3.24 30.689 20739.91593 1
141 1 6.9 3.1 5.1 2.3 5.29 34.891 15648.51349 1
53 1 5.5 2.3 4.0 1.3 1.69 16.167 5039.14375 0
... ... ... ... ... ... ... ... ... ...
8 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
49 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
112 1 6.8 3.0 5.5 2.1 4.41 32.500 14547.83568 1
17 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
15 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
104 1 6.5 3.0 5.8 2.2 4.84 32.800 11611.70625 1
91 1 6.1 3.0 4.6 1.4 1.96 31.600 8453.56301 0
46 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
43 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
101 1 5.8 2.7 5.1 1.9 3.61 24.783 6571.36768 1
37 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
38 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
125 1 7.2 3.2 6.0 1.8 3.24 38.768 19358.37632 1
103 1 6.3 2.9 5.6 1.8 3.24 29.989 9932.86543 1
71 1 6.1 2.8 4.0 1.3 1.69 25.952 8452.76301 0
65 1 6.7 3.1 4.4 1.4 1.96 34.191 13508.75107 0
149 1 5.9 3.0 5.1 1.8 3.24 32.100 7157.34299 1
40 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
5 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
131 1 7.9 3.8 6.4 2.0 4.00 61.272 30780.76399 1
57 1 4.9 2.4 3.3 1.0 1.00 17.124 2830.45249 0
30 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
148 1 6.2 3.4 5.4 2.3 5.29 44.704 9170.12832 1
132 1 6.4 2.8 5.6 2.2 4.84 27.552 10745.81824 1
22 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
29 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
50 1 7.0 3.2 4.7 1.4 1.96 37.468 16814.90000 0
140 1 6.7 3.1 5.6 2.4 5.76 35.391 13509.95107 1
64 1 5.6 2.9 3.6 1.3 1.69 27.989 5513.81776 0
108 1 6.7 2.5 5.8 1.8 3.24 21.425 13509.55107 1

90 rows × 9 columns

In [103]:
train=train.to_numpy()
In [104]:
w=np.ones(8)

(weight,cost)=lgd(0.001,train[:,0:8],train[:,8],w,300)
print weight,cost
[0.94772753 0.94770872 0.94772514 0.94770891 0.94772054 0.9477029
 0.94753766 0.59298975] [0.33693372 0.33570091 0.3344807  0.33327296 0.33207755 0.33089437
 0.32972326 0.32856411 0.3274168  0.3262812  0.32515719 0.32404465
 0.32294346 0.3218535  0.32077466 0.31970681 0.31864985 0.31760367
 0.31656814 0.31554316 0.31452862 0.31352441 0.31253042 0.31154655
 0.31057269 0.30960874 0.30865459 0.30771015 0.30677531 0.30584997
 0.30493403 0.3040274  0.30312998 0.30224167 0.30136238 0.30049202
 0.29963049 0.2987777  0.29793356 0.29709798 0.29627088 0.29545215
 0.29464173 0.29383952 0.29304543 0.29225939 0.29148131 0.2907111
 0.28994869 0.289194   0.28844694 0.28770744 0.28697542 0.2862508
 0.28553351 0.28482346 0.28412059 0.28342483 0.28273609 0.28205431
 0.28137941 0.28071133 0.28004999 0.27939533 0.27874728 0.27810576
 0.27747071 0.27684208 0.27621978 0.27560375 0.27499394 0.27439028
 0.2737927  0.27320114 0.27261555 0.27203586 0.27146201 0.27089394
 0.27033159 0.26977491 0.26922383 0.2686783  0.26813827 0.26760368
 0.26707447 0.26655059 0.26603198 0.2655186  0.26501038 0.26450728
 0.26400925 0.26351623 0.26302817 0.26254503 0.26206675 0.26159328
 0.26112459 0.26066061 0.26020131 0.25974663 0.25929653 0.25885096
 0.25840988 0.25797325 0.25754101 0.25711313 0.25668956 0.25627026
 0.25585519 0.2554443  0.25503755 0.25463491 0.25423633 0.25384176
 0.25345118 0.25306455 0.25268181 0.25230294 0.2519279  0.25155664
 0.25118914 0.25082535 0.25046524 0.25010877 0.24975591 0.24940662
 0.24906087 0.24871861 0.24837983 0.24804448 0.24771253 0.24738395
 0.2470587  0.24673675 0.24641807 0.24610264 0.24579041 0.24548135
 0.24517544 0.24487264 0.24457293 0.24427628 0.24398265 0.24369201
 0.24340434 0.24311962 0.2428378  0.24255886 0.24228278 0.24200953
 0.24173908 0.24147141 0.24120648 0.24094427 0.24068476 0.24042791
 0.24017372 0.23992214 0.23967315 0.23942674 0.23918287 0.23894152
 0.23870267 0.23846629 0.23823237 0.23800087 0.23777178 0.23754506
 0.23732071 0.2370987  0.236879   0.23666159 0.23644646 0.23623357
 0.23602292 0.23581448 0.23560822 0.23540414 0.2352022  0.23500239
 0.23480469 0.23460907 0.23441553 0.23422403 0.23403457 0.23384712
 0.23366166 0.23347818 0.23329665 0.23311707 0.2329394  0.23276364
 0.23258976 0.23241776 0.2322476  0.23207928 0.23191278 0.23174807
 0.23158516 0.23142401 0.23126461 0.23110695 0.23095101 0.23079678
 0.23064423 0.23049336 0.23034414 0.23019658 0.23005064 0.22990631
 0.22976358 0.22962244 0.22948287 0.22934485 0.22920838 0.22907344
 0.22894001 0.22880808 0.22867765 0.22854868 0.22842118 0.22829512
 0.2281705  0.22804731 0.22792552 0.22780513 0.22768612 0.22756849
 0.22745222 0.22733729 0.22722371 0.22711144 0.22700049 0.22689084
 0.22678248 0.22667539 0.22656958 0.22646502 0.2263617  0.22625961
 0.22615875 0.2260591  0.22596066 0.2258634  0.22576732 0.22567241
 0.22557867 0.22548607 0.22539461 0.22530428 0.22521507 0.22512697
 0.22503997 0.22495406 0.22486923 0.22478547 0.22470277 0.22462113
 0.22454053 0.22446097 0.22438243 0.22430491 0.2242284  0.22415288
 0.22407836 0.22400482 0.22393225 0.22386065 0.22379001 0.22372031
 0.22365156 0.22358373 0.22351683 0.22345085 0.22338578 0.22332161
 0.22325833 0.22319593 0.22313442 0.22307377 0.22301399 0.22295507
 0.22289699 0.22283975 0.22278335 0.22272777 0.22267302 0.22261907
 0.22256593 0.2225136  0.22246205 0.22241129 0.22236131 0.2223121
 0.22226365 0.22221597 0.22216904 0.22212285 0.22207741 0.2220327 ]

j=0.2220327

1 vs 3

In [105]:
data=pd.read_csv('data4 .csv')
data = pd.DataFrame(data) 
data.insert(0, "constant", [1]*150, True) 
train=data.sample(frac=0.6,random_state=200) #random state is a seed value
test=data.drop(train.index)
train[train['output']==2]=0
train['output'] = (train['output']==3) * 1
train['output']
train
Out[105]:
constant f_1 f_2 f_3 f_4 f_5 f_6 f_7 output
84 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
122 1 7.7 2.8 6.7 2.0 4.00 28.652 27077.34157 1
28 1 5.2 3.4 1.4 0.2 0.04 40.704 3806.84032 0
24 1 4.8 3.4 1.9 0.2 0.04 41.204 2553.33968 0
75 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
109 1 7.2 3.6 6.1 2.5 6.25 52.756 19358.87632 1
81 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
98 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
80 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
100 1 6.3 3.3 6.0 2.5 6.25 41.937 9933.66543 1
124 1 6.7 3.3 5.7 2.1 4.41 41.637 13510.25107 1
2 1 4.7 3.2 1.3 0.2 0.04 34.068 2297.95007 0
34 1 4.9 3.1 1.5 0.1 0.01 31.291 2829.35249 0
44 1 5.1 3.8 1.9 0.4 0.16 56.772 3455.95251 0
128 1 6.4 2.8 5.6 2.1 4.41 27.552 10745.81824 1
13 1 4.3 3.0 1.1 0.1 0.01 28.100 1474.18443 0
93 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
41 1 4.5 2.3 1.3 0.3 0.09 13.467 1848.88125 0
63 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
137 1 6.4 3.1 5.5 1.8 3.24 35.291 10746.01824 1
19 1 5.1 3.8 1.5 0.3 0.09 56.372 3455.55251 0
60 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
126 1 6.2 2.8 4.8 1.8 3.24 26.752 9168.92832 1
11 1 4.8 3.4 1.6 0.2 0.04 40.904 2553.03968 0
47 1 4.6 3.2 1.4 0.2 0.04 34.168 2064.22976 0
18 1 5.7 3.8 1.7 0.3 0.09 56.572 6022.42057 0
96 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
107 1 7.3 2.9 6.3 1.8 3.24 30.689 20739.91593 1
141 1 6.9 3.1 5.1 2.3 5.29 34.891 15648.51349 1
53 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
... ... ... ... ... ... ... ... ... ...
8 1 4.4 2.9 1.4 0.2 0.04 25.789 1653.46224 0
49 1 5.0 3.3 1.4 0.2 0.04 37.337 3129.70000 0
112 1 6.8 3.0 5.5 2.1 4.41 32.500 14547.83568 1
17 1 5.1 3.5 1.4 0.3 0.09 44.275 3455.15251 0
15 1 5.7 4.4 1.5 0.4 0.16 86.684 6022.82057 0
104 1 6.5 3.0 5.8 2.2 4.84 32.800 11611.70625 1
91 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
46 1 5.1 3.8 1.6 0.2 0.04 56.472 3455.65251 0
43 1 5.0 3.5 1.6 0.6 0.36 44.475 3130.10000 0
101 1 5.8 2.7 5.1 1.9 3.61 24.783 6571.36768 1
37 1 4.9 3.1 1.5 0.1 0.01 31.291 2829.35249 0
38 1 4.4 3.0 1.3 0.2 0.04 28.300 1653.46224 0
125 1 7.2 3.2 6.0 1.8 3.24 38.768 19358.37632 1
103 1 6.3 2.9 5.6 1.8 3.24 29.989 9932.86543 1
71 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
65 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
149 1 5.9 3.0 5.1 1.8 3.24 32.100 7157.34299 1
40 1 5.0 3.5 1.3 0.3 0.09 44.175 3129.80000 0
5 1 5.4 3.9 1.7 0.4 0.16 61.019 4597.25024 0
131 1 7.9 3.8 6.4 2.0 4.00 61.272 30780.76399 1
57 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
30 1 4.8 3.1 1.6 0.2 0.04 31.391 2552.73968 0
148 1 6.2 3.4 5.4 2.3 5.29 44.704 9170.12832 1
132 1 6.4 2.8 5.6 2.2 4.84 27.552 10745.81824 1
22 1 4.6 3.6 1.0 0.2 0.04 47.656 2064.22976 0
29 1 4.7 3.2 1.6 0.2 0.04 34.368 2298.25007 0
50 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
140 1 6.7 3.1 5.6 2.4 5.76 35.391 13509.95107 1
64 0 0.0 0.0 0.0 0.0 0.00 0.000 0.00000 0
108 1 6.7 2.5 5.8 1.8 3.24 21.425 13509.55107 1

90 rows × 9 columns

In [106]:
train=train.to_numpy()
In [107]:
w=np.ones(8)

(weight,cost)=lgd(0.001,train[:,0:8],train[:,8],w,300)
print weight,cost
[0.95180076 0.95183009 0.95184883 0.951732   0.95175638 0.95171087
 0.95250465 0.53636707] [0.19610408 0.19575006 0.19540014 0.19505427 0.1947124  0.1943745
 0.1940405  0.19371037 0.19338406 0.19306153 0.19274272 0.19242761
 0.19211614 0.19180827 0.19150396 0.19120318 0.19090587 0.190612
 0.19032153 0.19003443 0.18975064 0.18947014 0.18919288 0.18891884
 0.18864796 0.18838022 0.18811559 0.18785401 0.18759547 0.18733993
 0.18708735 0.18683769 0.18659094 0.18634704 0.18610598 0.18586772
 0.18563222 0.18539947 0.18516942 0.18494204 0.18471732 0.18449521
 0.18427569 0.18405873 0.1838443  0.18363237 0.18342293 0.18321593
 0.18301135 0.18280917 0.18260935 0.18241188 0.18221673 0.18202387
 0.18183328 0.18164493 0.1814588  0.18127487 0.18109311 0.18091349
 0.180736   0.18056061 0.1803873  0.18021605 0.18004683 0.17987962
 0.17971441 0.17955117 0.17938988 0.17923051 0.17907306 0.17891749
 0.17876379 0.17861195 0.17846193 0.17831372 0.1781673  0.17802265
 0.17787976 0.1777386  0.17759917 0.17746143 0.17732537 0.17719098
 0.17705823 0.17692712 0.17679762 0.17666971 0.17654339 0.17641863
 0.17629542 0.17617374 0.17605358 0.17593492 0.17581774 0.17570204
 0.17558779 0.17547498 0.1753636  0.17525364 0.17514507 0.17503788
 0.17493206 0.1748276  0.17472449 0.1746227  0.17452222 0.17442305
 0.17432517 0.17422857 0.17413323 0.17403914 0.17394629 0.17385467
 0.17376426 0.17367506 0.17358705 0.17350021 0.17341455 0.17333004
 0.17324667 0.17316444 0.17308333 0.17300334 0.17292444 0.17284663
 0.17276991 0.17269425 0.17261965 0.1725461  0.17247359 0.17240211
 0.17233164 0.17226219 0.17219373 0.17212626 0.17205978 0.17199426
 0.1719297  0.1718661  0.17180344 0.17174172 0.17168092 0.17162103
 0.17156206 0.17150398 0.17144679 0.17139049 0.17133506 0.1712805
 0.17122679 0.17117394 0.17112192 0.17107074 0.17102039 0.17097085
 0.17092213 0.17087421 0.17082708 0.17078074 0.17073518 0.1706904
 0.17064638 0.17060313 0.17056062 0.17051886 0.17047784 0.17043756
 0.17039799 0.17035915 0.17032102 0.17028359 0.17024686 0.17021083
 0.17017548 0.17014081 0.17010681 0.17007349 0.17004082 0.17000881
 0.16997745 0.16994674 0.16991666 0.16988722 0.1698584  0.1698302
 0.16980262 0.16977565 0.16974928 0.16972351 0.16969834 0.16967375
 0.16964975 0.16962632 0.16960347 0.16958118 0.16955946 0.16953829
 0.16951768 0.16949762 0.1694781  0.16945911 0.16944066 0.16942274
 0.16940535 0.16938847 0.1693721  0.16935625 0.1693409  0.16932606
 0.16931171 0.16929785 0.16928449 0.1692716  0.1692592  0.16924727
 0.16923581 0.16922482 0.1692143  0.16920423 0.16919462 0.16918546
 0.16917675 0.16916848 0.16916065 0.16915325 0.16914629 0.16913976
 0.16913365 0.16912796 0.1691227  0.16911784 0.16911339 0.16910936
 0.16910572 0.16910248 0.16909965 0.1690972  0.16909514 0.16909347
 0.16909219 0.16909128 0.16909075 0.16909059 0.16909081 0.16909139
 0.16909233 0.16909363 0.1690953  0.16909731 0.16909968 0.1691024
 0.16910547 0.16910887 0.16911262 0.1691167  0.16912112 0.16912587
 0.16913095 0.16913635 0.16914208 0.16914813 0.1691545  0.16916118
 0.16916817 0.16917547 0.16918309 0.169191   0.16919922 0.16920774
 0.16921655 0.16922566 0.16923506 0.16924475 0.16925473 0.169265
 0.16927554 0.16928637 0.16929747 0.16930886 0.16932051 0.16933243
 0.16934463 0.16935709 0.16936981 0.1693828  0.16939605 0.16940955
 0.16942332 0.16943733 0.1694516  0.16946611 0.16948088 0.16949589
 0.16951114 0.16952664 0.16954237 0.16955834 0.16957455 0.16959099]

j=0.16959099

As third case have least J it will be used to predict

In [108]:
h_hat=np.zeros(90)
weight=[0.95180076 ,0.95183009, 0.95184883, 0.951732,   0.95175638 ,0.95171087,
 0.95250465, 0.53636707]
for i in range(90):
    for j in range(8):
        h_hat[i]+=weight[j]*train[i][j]
    
fig=plt.figure(figsize=(8, 4))
ax=fig.add_subplot(111)
colors = ['#483d8b', '#cc8400','red','green']
colors_data = [colors[int(i)] for i in train[:,8]]
ax.scatter(train[:,1],train[:,2], color=colors_data, s=100, edgecolor='black', linewidth=2, alpha=0.7)
ax.plot(h_hat, linewidth=2, color='#483d8b')
plt.xlim([4, 7])
plt.ylim([0, 20])
Out[108]:
(0, 20)
In [109]:
def predict(feature,weight):
    
    m=test.shape[0]
    n=(test.shape[1])-1
    predicted=np.zeros(m)
    y=np.zeros(m)
    
    for i in range(m):
        h=0
        for j in range(n):
            h+=weight[j]*feature[i][j]
        predicted[i]=sigmoid(h)
        if(predicted[i]>=0.5):
            y[i]=1
        else:
            y[i]=0
        
    return y
In [111]:
test=test.to_numpy()
In [112]:
for i in range(40):
    if(test[i,5]==y[i] and test[i,5]==1):
        u1+=1
    if(test[i,5]==y[i] and test[i,5]==2):
        u2+=1
    if(test[i,5]==y[i] and test[i,5]==3):
        u3+=1  
individual_acc_1=float(u1)/(u1+u2+u3)
individual_acc_2=float(u2)/(u1+u2+u3)
individual_acc_3=float(u3)/(u1+u2+u3)
overall_accuracy=float(u1+u2+u3)/40
print individual_acc_1,individual_acc_2,individual_acc_3
print overall_accuracy
1.0 0.0 0.0
0.1

Clustering

In [113]:
feature=pd.read_csv('data2.csv')
In [114]:
plt.scatter(feature['x1'],feature['x2'],feature['x3'],feature['x4'])
Out[114]:
<matplotlib.collections.PathCollection at 0x7fe012b68b50>
In [115]:
from math import sqrt
import random
def feature_normalize(data):
    data=(data-data.mean())/data.std()
    return data
In [116]:
feature=feature.to_numpy()
def cluster(K,feature,iteration):
    #centroids=[[5, 3.4 ,1.5, 0.2], [6, 3 , 4 , 1.4] ,[7, 3, 6 ,2]]
    centroids=[(random.sample(range(-1,5), 4)),(random.sample(range(-1,5), 4)),(random.sample(range(-1,5), 4))]
    print centroids
    diff=[0,0,0]
    m=feature.shape[0]
    n=feature.shape[1]
    
    
    
    for it in range(iteration):
        
        clusters_1=[]
        clusters_2=[]
        clusters_3=[]
        c=0
        for i in range(m):
                
                for k in range(K):
                    diff[k]=sqrt(np.dot((feature[i]-centroids[k]),(feature[i]-centroids[k])))
                   
                c=diff.index(min(diff))
                        
                
                if(c==0):
                    (clusters_1).append(feature[i])
                elif (c==1):
                    (clusters_2).append(feature[i])
                else:
                    (clusters_3).append(feature[i])
                
        for k in range(K):
            if(k==0):
                centroids[k]=sum(clusters_1)/len(clusters_1)
            elif (k==1):
                centroids[k]=sum(clusters_2)/len(clusters_2)
            else:
                centroids[k]=sum(clusters_3)/len(clusters_3)
            
    len(clusters_2),len(clusters_3)
    
    return centroids,clusters_2,clusters_1,clusters_3
    
In [149]:
(centroids,cluster_2,cluster_1,clusters_3)=cluster(3,feature,1000)
print centroids
[[2, 1, 0, 3], [1, 4, 3, 0], [1, 4, 2, 3]]
[array([5.006, 3.418, 1.464, 0.244]), array([5.88360656, 2.74098361, 4.38852459, 1.43442623]), array([6.85384615, 3.07692308, 5.71538462, 2.05384615])]
In [150]:
fig= plt.figure(figsize=(10,9))

plt.scatter(5.006, 3.418, s=200, c='g', marker='s')
plt.scatter(5.88360656, 2.74098361, s=200, c='r', marker='s')
plt.scatter(6.85384615, 3.07692308, s=200, c='b', marker='s')


plt.scatter(feature[ : , 0], feature[ : , 1], s =50, c='g')
plt.scatter(feature[ : , 2], feature[ : , 3], s =50, c='g')
#blue red green squares represent the centroids
Out[150]:
<matplotlib.collections.PathCollection at 0x7fe012ac7710>

Multiple regression using 5 hold cross validation

In [151]:
from random import seed
from random import randrange
from csv import reader
from math import exp
 
# Load a CSV file
def load_csv(filename):
    dataset = list()
    with open(filename, 'r') as file:
        csv_reader = reader(file)
        for row in csv_reader:
            if not row:
                continue
            dataset.append(row)
    return dataset
 
# Convert string column to float
def str_column_to_float(dataset, column):
    for row in dataset:
        row[column] = float(row[column].strip())
 
# Find the min and max values for each column
def dataset_minmax(dataset):
    minmax = list()
    for i in range(len(dataset[0])):
        col_values = [row[i] for row in dataset]
        value_min = min(col_values)
        value_max = max(col_values)
        minmax.append([value_min, value_max])
    return minmax
 
# Rescale dataset columns to the range 0-1
def normalize_dataset(dataset, minmax):
    for row in dataset:
        for i in range(len(row)):
            row[i] = (row[i] - minmax[i][0]) / (minmax[i][1] - minmax[i][0])
 
# Split a dataset into k folds
def cross_validation_split(dataset, n_folds):
    dataset_split = list()
    dataset_copy = list(dataset)
    fold_size = int(len(dataset) / n_folds)
    for i in range(n_folds):
        fold = list()
        while len(fold) < fold_size:
            index = randrange(len(dataset_copy))
            fold.append(dataset_copy.pop(index))
        dataset_split.append(fold)
    return dataset_split
 
# Calculate accuracy percentage
def accuracy_metric(actual, predicted):
    correct = 0
    for i in range(len(actual)):
        if actual[i] == predicted[i]:
            correct += 1
    return correct / float(len(actual)) * 100.0
 
# Evaluate an algorithm using a cross validation split
def evaluate_algorithm(dataset, algorithm, n_folds, *args):
    folds = cross_validation_split(dataset, n_folds)
    scores = list()
    for fold in folds:
        train_set = list(folds)
        train_set.remove(fold)
        train_set = sum(train_set, [])
        test_set = list()
        for row in fold:
            row_copy = list(row)
            test_set.append(row_copy)
            row_copy[-1] = None
        predicted = algorithm(train_set, test_set, *args)
        actual = [row[-1] for row in fold]
        accuracy = accuracy_metric(actual, predicted)
        scores.append(accuracy)
    return scores
 
# Make a prediction with coefficients
def predict(row, coefficients):
    yhat = coefficients[0]
    for i in range(len(row)-1):
        yhat += coefficients[i + 1] * row[i]
    return 1.0 / (1.0 + exp(-yhat))
 
# Estimate logistic regression coefficients using stochastic gradient descent
def coefficients_sgd(train, l_rate, n_epoch):
    coef = [0.0 for i in range(len(train[0]))]
    for epoch in range(n_epoch):
        for row in train:
            yhat = predict(row, coef)
            error = row[-1] - yhat
            coef[0] = coef[0] + l_rate * error * yhat * (1.0 - yhat)
            for i in range(len(row)-1):
                coef[i + 1] = coef[i + 1] + l_rate * error * yhat * (1.0 - yhat) * row[i]
    return coef
 
# Linear Regression Algorithm With Stochastic Gradient Descent
def logistic_regression(train, test, l_rate, n_epoch):
    predictions = list()
    coef = coefficients_sgd(train, l_rate, n_epoch)
    for row in test:
        yhat = predict(row, coef)
        yhat = round(yhat)
        predictions.append(yhat)
    return(predictions)
 
# Test the logistic regression algorithm 
seed(1)
# load and prepare data
filename = 'data4 .csv'

dataset = load_csv(filename)
dataset.pop(0)
for i in range(len(dataset[0])):
    str_column_to_float(dataset, i)
# normalize
minmax = dataset_minmax(dataset)
normalize_dataset(dataset, minmax)
# evaluate algorithm
n_folds = 5
l_rate = 0.1
n_epoch = 100
scores = evaluate_algorithm(dataset, logistic_regression, n_folds, l_rate, n_epoch)
print('Scores: %s' % scores)
print('Mean Accuracy: %.3f%%' % (sum(scores)/float(len(scores))))
Scores: [66.66666666666666, 70.0, 50.0, 66.66666666666666, 80.0]
Mean Accuracy: 66.667%

Likelihood ratio test

In [152]:
df=pd.read_csv("data3.csv")
In [153]:
df = pd.DataFrame(df) 
df.insert(0, "constant", [1]*100, True) 
df
Out[153]:
constant w1 w2 w3 w4 t
0 1 5.1 3.5 1.4 0.2 1
1 1 4.9 3.0 1.4 0.2 1
2 1 4.7 3.2 1.3 0.2 1
3 1 4.6 3.1 1.5 0.2 1
4 1 5.0 3.6 1.4 0.2 1
5 1 5.4 3.9 1.7 0.4 1
6 1 4.6 3.4 1.4 0.3 1
7 1 5.0 3.4 1.5 0.2 1
8 1 4.4 2.9 1.4 0.2 1
9 1 4.9 3.1 1.5 0.1 1
10 1 5.4 3.7 1.5 0.2 1
11 1 4.8 3.4 1.6 0.2 1
12 1 4.8 3.0 1.4 0.1 1
13 1 4.3 3.0 1.1 0.1 1
14 1 5.8 4.0 1.2 0.2 1
15 1 5.7 4.4 1.5 0.4 1
16 1 5.4 3.9 1.3 0.4 1
17 1 5.1 3.5 1.4 0.3 1
18 1 5.7 3.8 1.7 0.3 1
19 1 5.1 3.8 1.5 0.3 1
20 1 5.4 3.4 1.7 0.2 1
21 1 5.1 3.7 1.5 0.4 1
22 1 4.6 3.6 1.0 0.2 1
23 1 5.1 3.3 1.7 0.5 1
24 1 4.8 3.4 1.9 0.2 1
25 1 5.0 3.0 1.6 0.2 1
26 1 5.0 3.4 1.6 0.4 1
27 1 5.2 3.5 1.5 0.2 1
28 1 5.2 3.4 1.4 0.2 1
29 1 4.7 3.2 1.6 0.2 1
... ... ... ... ... ... ...
70 1 5.9 3.2 4.8 1.8 2
71 1 6.1 2.8 4.0 1.3 2
72 1 6.3 2.5 4.9 1.5 2
73 1 6.1 2.8 4.7 1.2 2
74 1 6.4 2.9 4.3 1.3 2
75 1 6.6 3.0 4.4 1.4 2
76 1 6.8 2.8 4.8 1.4 2
77 1 6.7 3.0 5.0 1.7 2
78 1 6.0 2.9 4.5 1.5 2
79 1 5.7 2.6 3.5 1.0 2
80 1 5.5 2.4 3.8 1.1 2
81 1 5.5 2.4 3.7 1.0 2
82 1 5.8 2.7 3.9 1.2 2
83 1 6.0 2.7 5.1 1.6 2
84 1 5.4 3.0 4.5 1.5 2
85 1 6.0 3.4 4.5 1.6 2
86 1 6.7 3.1 4.7 1.5 2
87 1 6.3 2.3 4.4 1.3 2
88 1 5.6 3.0 4.1 1.3 2
89 1 5.5 2.5 4.0 1.3 2
90 1 5.5 2.6 4.4 1.2 2
91 1 6.1 3.0 4.6 1.4 2
92 1 5.8 2.6 4.0 1.2 2
93 1 5.0 2.3 3.3 1.0 2
94 1 5.6 2.7 4.2 1.3 2
95 1 5.7 3.0 4.2 1.2 2
96 1 5.7 2.9 4.2 1.3 2
97 1 6.2 2.9 4.3 1.3 2
98 1 5.1 2.5 3.0 1.1 2
99 1 5.7 2.8 4.1 1.3 2

100 rows × 6 columns

In [154]:
train=df.sample(frac=0.6,random_state=200) #random state is a seed value
test=df.drop(train.index)
len(train)
train['t'] = (train['t'] < 2)*1
train=train.to_numpy()

len(test)
Out[154]:
40
In [ ]:
 
In [155]:
def sigmoid(x):
    y=(1/(1+np.exp(-x)))
    return y
In [156]:
def feature_normalize(data):
    data=(data-data.mean())/data.std()
    return data
In [157]:
def cost_function(feature,output,weight):
    m=train.shape[0]
    n=1
    cost=np.ones(m)
    cost_func=0
    for i in range(m):
        
        
        h=weight[0]*1
        cost[i]=-((output[i])*np.log(sigmoid(h)))-((1-output[i])*np.log(1-sigmoid(h)))
    for k in range(m):
        cost_func+=cost[k]
    net_cost=cost_func/m
    return net_cost
In [158]:
def lgd(alpha,feature,output,weights,iteration):
    
    
    cost=np.ones(iteration)
    m=train.shape[0]
    n=1
    weights=np.ones(n)
    
    for it in range(iteration):
        temp=weights
        gradient=np.ones(n)
        for i in range(m):
            
            
                
            h=temp[0]*1
            error=output[i]-h
            
            
            gradient[0]+=error*1
                
        
        temp[0]=temp[0]+(alpha*gradient[0])/m
        weights=temp
        
        cost[it]=cost_function(feature,output,weights)
    plt.plot(cost)
    return weights,cost
In [159]:
test=test.to_numpy()

from previous question log l=30.6

In [161]:
w=np.ones(5)
(weight,cost)=lgd(0.08,train[:,0],train[:,5],w,500)
print weight
[0.53333333]

2 log l = 87

Only intercept model is best

In [162]:
def predict(feature,weight):
    
    m=test.shape[0]
    n=1
    predicted=np.zeros(m)
    y=np.zeros(m)
    chi_sq=0
    h=0
    for i in range(m):
        h+=weight[0]*1
        predicted[i]=sigmoid(h)
        print predicted[i]
        if(predicted[i]>=0.5):
            y[i]=2
        else:
            y[i]=1
        chi_sq+=((predicted[i]-test[i,5])**2)/predicted[i]
    return y,predicted,chi_sq
    
y,predicted,chi_sq=(predict(test, weight))
print y,chi_sq
0.6302602229177514
0.7439624913247582
0.8320183851339248
0.8941010377640823
0.9350308308713361
0.9608342772032357
0.9766454835230229
0.9861656084433474
0.9918374288468401
0.9951952471128405
0.9971757006472715
0.9983411989198255
0.9990262029122943
0.9994284955764196
0.9996646498695336
0.9998032408393402
0.9998845625935888
0.9999322758503804
0.9999602687714186
0.9999766914421585
0.9999863259909154
0.9999919781489238
0.9999952939957882
0.9999972392350497
0.9999983804058309
0.9999990498705674
0.9999994426099414
0.9999996730091539
0.9999998081720304
0.9999998874648379
0.9999999339816695
0.999999961270595
0.9999999772795406
0.9999999866711282
0.9999999921806678
0.9999999954128183
0.9999999973089471
0.9999999984213039
0.9999999990738639
0.9999999994566857
[2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2.
 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2.] 21.358615549771724
In [163]:
m=40
true_positive=0
true_negative=0
false_positive=0
false_negative=0
for i in range(m):
    if(test[i,5]==y[i] and int(y[i])==2):
        true_negative+=1
    if(test[i,5]==y[i] and int(y[i])==1):
        true_positive+=1
    if(test[i,5]!=y[i] and int(y[i])==2):
        false_negative+=1
    if(test[i,5]!=y[i] and int(y[i])==1):
        false_positive+=1
sensitivity=(float(true_positive))/(true_positive+false_negative)
specificity=(float(true_negative))/(true_negative+false_positive)
accuracy=float(true_positive+true_negative)/m
print sensitivity,specificity,accuracy
print true_positive,true_negative,false_positive,false_negative
0.0 1.0 0.525
0 21 0 19

Maximum a Posterior Probability

In [164]:
# Naive Bayes 
from csv import reader
from random import seed
from random import randrange
from math import sqrt
from math import exp
from math import pi

# Load a CSV file
def load_csv(filename):
    dataset = list()
    with open(filename, 'r') as file:
        
        csv_reader = reader(file)
        for row in csv_reader:
            if not row:
                continue
            dataset.append(row)
    return dataset

# Convert string column to float
def str_column_to_float(dataset, column):
    for row in dataset:
        row[column] = float(row[column].strip())

# Convert string column to integer
def str_column_to_int(dataset, column):
    class_values = [row[column] for row in dataset]
    unique = set(class_values)
    lookup = dict()
    for i, value in enumerate(unique):
        lookup[value] = i
    for row in dataset:
        row[column] = lookup[row[column]]
    return lookup


# Calculate accuracy percentage
def accuracy_metric(actual, predicted):
    correct = 0
    for i in range(len(actual)):
        if actual[i] == predicted[i]:
            correct += 1
    return correct / float(len(actual)) * 100.0

# Split a dataset into k folds
def cross_validation_split(dataset, n_folds):
    dataset_split = list()
    dataset_copy = list(dataset)
    fold_size = int(len(dataset) / n_folds)
    for _ in range(n_folds):
        fold = list()
        while len(fold) < fold_size:
            index = randrange(len(dataset_copy))
            fold.append(dataset_copy.pop(index))
        dataset_split.append(fold)
    return dataset_split


# Evaluate an algorithm using a cross validation split
def evaluate_algorithm(dataset, algorithm, n_folds, *args):
    folds = cross_validation_split(dataset, n_folds)
    scores = list()
    for fold in folds:
        train_set = list(folds)
        train_set.remove(fold)
        train_set = sum(train_set, [])
        test_set = list()
        for row in fold:
            row_copy = list(row)
            test_set.append(row_copy)
            row_copy[-1] = None
        predicted = algorithm(train_set, test_set, *args)
        actual = [row[-1] for row in fold]
        accuracy = accuracy_metric(actual, predicted)
        scores.append(accuracy)
    return scores

# Split the dataset by class values, returns a dictionary
def separate_by_class(dataset):
    separated = dict()
    for i in range(len(dataset)):
        vector = dataset[i]
        class_value = vector[-1]
        if (class_value not in separated):
            separated[class_value] = list()
        separated[class_value].append(vector)
    return separated

# Calculate the mean of a list of numbers
def mean(numbers):
    return sum(numbers)/float(len(numbers))

# Calculate the standard deviation of a list of numbers
def stdev(numbers):
    avg = mean(numbers)
    variance = sum([(x-avg)**2 for x in numbers]) / float(len(numbers)-1)
    return sqrt(variance)

# Calculate the mean, stdev and count for each column in a dataset
def summarize_dataset(dataset):
    summaries = [(mean(column), stdev(column), len(column)) for column in zip(*dataset)]
    del(summaries[-1])
    return summaries

# Split dataset by class then calculate statistics for each row
def summarize_by_class(dataset):
    separated = separate_by_class(dataset)
    summaries = dict()
    for class_value, rows in separated.items():
        summaries[class_value] = summarize_dataset(rows)
    return summaries

# Calculate the Gaussian probability distribution function for x
def calculate_probability(x, mean, stdev):
    exponent = exp(-((x-mean)**2 / (2 * stdev**2 )))
    return (1 / (sqrt(2 * pi) * stdev)) * exponent

# Calculate the probabilities of predicting each class for a given row
def calculate_class_probabilities(summaries, row):
    total_rows = sum([summaries[label][0][2] for label in summaries])
    probabilities = dict()
    for class_value, class_summaries in summaries.items():
        probabilities[class_value] = summaries[class_value][0][2]/float(total_rows)
        for i in range(len(class_summaries)):
            mean, stdev, _ = class_summaries[i]
            probabilities[class_value]*= calculate_probability(row[i], mean, stdev)
    return probabilities

# Predict the class for a given row
def predict(summaries, row):
    probabilities = calculate_class_probabilities(summaries, row)
    best_label, best_prob = None, -1
    for class_value, probability in probabilities.items():
        if best_label is None or probability > best_prob:
            best_prob = probability
            best_label = class_value
    return best_label

# Naive Bayes Algorithm
def naive_bayes(train, test):
    summarize = summarize_by_class(train)
    predictions = list()
    for row in test:
        output = predict(summarize, row)
        predictions.append(output)
    return(predictions)

# Test Naive Bayes 
seed(1)
filename="data4 .csv"

dataset = load_csv(filename)
dataset.pop(0)
print(dataset[0])
for i in range(len(dataset[0])-1):
    str_column_to_float(dataset, i)
# convert class column to integers
str_column_to_int(dataset, len(dataset[0])-1)
# evaluate algorithm
n_folds = 2
scores = evaluate_algorithm(dataset, naive_bayes, n_folds)
print('Scores: %s' % scores)
print('Mean Accuracy: %.3f%%' % (sum(scores)/float(len(scores))))
['5.1', '3.5', '1.4', '0.2', '0.04', '44.275', '3455.15251', '1']
Scores: [94.66666666666667, 94.66666666666667]
Mean Accuracy: 94.667%

Maximum Likelihood Estimation Decision

In [165]:
# Naive Bayes 
from csv import reader
from random import seed
from random import randrange
from math import sqrt
from math import exp
from math import pi

# Load a CSV file
def load_csv(filename):
    dataset = list()
    with open(filename, 'r') as file:
        
        csv_reader = reader(file)
        for row in csv_reader:
            if not row:
                continue
            dataset.append(row)
    return dataset

# Convert string column to float
def str_column_to_float(dataset, column):
    for row in dataset:
        row[column] = float(row[column].strip())

# Convert string column to integer
def str_column_to_int(dataset, column):
    class_values = [row[column] for row in dataset]
    unique = set(class_values)
    lookup = dict()
    for i, value in enumerate(unique):
        lookup[value] = i
    for row in dataset:
        row[column] = lookup[row[column]]
    return lookup


# Calculate accuracy percentage
def accuracy_metric(actual, predicted):
    correct = 0
    for i in range(len(actual)):
        if actual[i] == predicted[i]:
            correct += 1
    return correct / float(len(actual)) * 100.0

# Split a dataset into k folds
def cross_validation_split(dataset, n_folds):
    dataset_split = list()
    dataset_copy = list(dataset)
    fold_size = int(len(dataset) / n_folds)
    for _ in range(n_folds):
        fold = list()
        while len(fold) < fold_size:
            index = randrange(len(dataset_copy))
            fold.append(dataset_copy.pop(index))
        dataset_split.append(fold)
    return dataset_split


# Evaluate an algorithm using a cross validation split
def evaluate_algorithm(dataset, algorithm, n_folds, *args):
    folds = cross_validation_split(dataset, n_folds)
    scores = list()
    for fold in folds:
        train_set = list(folds)
        train_set.remove(fold)
        train_set = sum(train_set, [])
        test_set = list()
        for row in fold:
            row_copy = list(row)
            test_set.append(row_copy)
            row_copy[-1] = None
        predicted = algorithm(train_set, test_set, *args)
        actual = [row[-1] for row in fold]
        accuracy = accuracy_metric(actual, predicted)
        scores.append(accuracy)
    return scores

# Split the dataset by class values, returns a dictionary
def separate_by_class(dataset):
    separated = dict()
    for i in range(len(dataset)):
        vector = dataset[i]
        class_value = vector[-1]
        if (class_value not in separated):
            separated[class_value] = list()
        separated[class_value].append(vector)
    return separated

# Calculate the mean of a list of numbers
def mean(numbers):
    return sum(numbers)/float(len(numbers))

# Calculate the standard deviation of a list of numbers
def stdev(numbers):
    avg = mean(numbers)
    variance = sum([(x-avg)**2 for x in numbers]) / float(len(numbers)-1)
    return sqrt(variance)

# Calculate the mean, stdev and count for each column in a dataset
def summarize_dataset(dataset):
    summaries = [(mean(column), stdev(column), len(column)) for column in zip(*dataset)]
    del(summaries[-1])
    return summaries

# Split dataset by class then calculate statistics for each row
def summarize_by_class(dataset):
    separated = separate_by_class(dataset)
    summaries = dict()
    for class_value, rows in separated.items():
        summaries[class_value] = summarize_dataset(rows)
    return summaries

# Calculate the Gaussian probability distribution function for x
def calculate_probability(x, mean, stdev):
    exponent = exp(-((x-mean)**2 / (2 * stdev**2 )))
    return (1 / (sqrt(2 * pi) * stdev)) * exponent

# Calculate the probabilities of predicting each class for a given row
def calculate_class_probabilities(summaries, row):
    total_rows = sum([summaries[label][0][2] for label in summaries])
    probabilities = dict()
    for class_value, class_summaries in summaries.items():
        probabilities[class_value] = summaries[class_value][0][2]/float(total_rows)
        for i in range(len(class_summaries)):
            mean, stdev, _ = class_summaries[i]
            probabilities[class_value]= calculate_probability(row[i], mean, stdev)
    return probabilities

# Predict the class for a given row
def predict(summaries, row):
    probabilities = calculate_class_probabilities(summaries, row)
    best_label, best_prob = None, -1
    for class_value, probability in probabilities.items():
        if best_label is None or probability > best_prob:
            best_prob = probability
            best_label = class_value
    return best_label

# Naive Bayes Algorithm
def naive_bayes(train, test):
    summarize = summarize_by_class(train)
    predictions = list()
    for row in test:
        output = predict(summarize, row)
        predictions.append(output)
    return(predictions)

# Test Naive Bayes 
seed(1)
filename="data4 .csv"

dataset = load_csv(filename)
dataset.pop(0)
print(dataset[0])
for i in range(len(dataset[0])-1):
    str_column_to_float(dataset, i)
# convert class column to integers
str_column_to_int(dataset, len(dataset[0])-1)
# evaluate algorithm
n_folds = 2
scores = evaluate_algorithm(dataset, naive_bayes, n_folds)
print('Scores: %s' % scores)
print('Mean Accuracy: %.3f%%' % (sum(scores)/float(len(scores))))
['5.1', '3.5', '1.4', '0.2', '0.04', '44.275', '3455.15251', '1']
Scores: [61.33333333333333, 68.0]
Mean Accuracy: 64.667%

Learning from assignment

I have learnt various operation on pandas like random splitting and adding columns etc. I have gained knowledge of algorithms and their implementations such as batch,stochastic and mini batch regression is used to get particular predicted value and with the help of rmse we can calculate error in prediction. Similary we can use logistic regression to predict yes or no based decision. Through clustering we can predict catagory of data with the help of centroids.And we can implement various probabilistic classifier also.

The End